diff --git a/.github/workflows/ci-script-tests.yml b/.github/workflows/ci-script-tests.yml new file mode 100644 index 000000000..c7ca4a2c8 --- /dev/null +++ b/.github/workflows/ci-script-tests.yml @@ -0,0 +1,47 @@ +name: CI script tests + +# Runs the Pester tests of the scripts in ci on Linux and Windows, where the +# builds run them. The environment variable names workflow runs its own test. +# The test of run-fresh-clone-example.ps1 also runs the calls in +# examples-fresh-clone.yml, so a change to that workflow runs the tests too. + +on: + pull_request: + paths: + - 'ci/**' + - '.github/workflows/ci-script-tests.yml' + # run-fresh-clone-example.Tests.ps1 runs the calls this workflow makes. + - '.github/workflows/examples-fresh-clone.yml' + push: + branches: + - main + - rc + paths: + - 'ci/**' + - '.github/workflows/ci-script-tests.yml' + - '.github/workflows/examples-fresh-clone.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + ci-script-tests: + name: Test the CI scripts on ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Run the tests + shell: pwsh + run: | + $tests = Get-ChildItem ./ci -Filter *.Tests.ps1 | + Where-Object Name -ne 'check-environment-variable-names.Tests.ps1' + Invoke-Pester -Path $tests.FullName -CI -Output Detailed diff --git a/.github/workflows/cloud-51did.yml b/.github/workflows/cloud-51did.yml index e99cdcb13..92566886b 100644 --- a/.github/workflows/cloud-51did.yml +++ b/.github/workflows/cloud-51did.yml @@ -7,6 +7,12 @@ name: Cloud 51Did Tests # the cloud for a 51Did and fail, saying so, when a key gets an answer with no # identifier in it. That is entitlement working rather than a fault. # +# Each run also runs the Cloud Request Engine's live test on a license key +# alone (pipeline/pipeline.cloudrequestengine, CloudRequestEngineLiveTests), +# with the cloud license in _51DEGREES_LICENSE_KEY_BESPOKE. Without one that +# test skips. _51DEGREES_LICENSE_KEY holds the license for the data files, so +# it is not set here. +# # The secrets are named one by one rather than passed as toJSON(secrets). # GitHub marks a workflow that hands every secret to a step as possibly # malicious and holds each run for manual approval, so the nightly never ran. @@ -29,9 +35,10 @@ on: schedule: - cron: '10 5 * * *' pull_request: - branches: [main] + branches: [main, rc] paths: - 'pipeline/pipeline.did/**' + - 'pipeline/pipeline.cloudrequestengine/**' - 'pipeline/owid-java/**' - '.github/workflows/cloud-51did.yml' @@ -55,13 +62,29 @@ jobs: shell: pwsh env: _51DEGREES_RESOURCE_KEY_BESPOKE: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - _51DEGREES_RESOURCE_KEY_51DID: ${{ vars._51DEGREES_RESOURCE_KEY_51DID }} + _51DEGREES_RESOURCE_KEY_51DID: ${{ secrets._51DEGREES_RESOURCE_KEY_51DID }} # Without a license key the tests that redeem a sealed verdict skip. # The service only redeems with a license key of the resource key's - # own account, so each one is paired by suffix with the resource - # key it belongs to. + # own account, so each resource key is paired by suffix with the + # license key it belongs to. Only the 51DID pass gets this one. _51DEGREES_LICENSE_KEY_51DID: ${{ secrets._51DEGREES_LICENSE_KEY_51DID }} + # The license for the cloud products, for the test on a license key + # alone. + _51DEGREES_LICENSE_KEY_BESPOKE: ${{ secrets._51DEGREES_LICENSE_KEY_BESPOKE }} run: | + # A license key whose resource key of the same suffix is empty means + # that secret is missing. Its licensed tests would then never run, + # and the job would still pass, so it fails here, naming it. + $missing = Get-ChildItem env: | + Where-Object { $_.Name -like '_51DEGREES_LICENSE_KEY_*' -and $_.Value } | + ForEach-Object { $_.Name -replace '^_51DEGREES_LICENSE_KEY', '_51DEGREES_RESOURCE_KEY' } | + Where-Object { -not [Environment]::GetEnvironmentVariable($_) } + if ($missing) { + throw ("The secret $($missing -join ', ') is empty or missing, " + + 'but the license key of the same suffix is set. Add the ' + + 'organization secret, a resource key of the same account as ' + + 'that license key.') + } $keys = Get-ChildItem env: | Where-Object { $_.Name -like '_51DEGREES_RESOURCE_KEY*' -and $_.Value } if (-not $keys) { @@ -71,22 +94,27 @@ jobs: 'from a fork gets no secrets, so run this workflow from a ' + 'branch in this repository.') } + # The 51Did tests redeem with _51DEGREES_LICENSE_KEY_51DID. A pass + # whose resource key is not paired with it by suffix redeems with no + # license, so those tests skip rather than come back unreadable. + $didLicense = $env:_51DEGREES_LICENSE_KEY_51DID $failed = $false foreach ($k in $keys) { - Write-Host "::group::DidClientLiveTests with $($k.Name)" + Write-Host "::group::Live tests with $($k.Name)" $env:_51DEGREES_RESOURCE_KEY = $k.Value $licenseName = $k.Name -replace '^_51DEGREES_RESOURCE_KEY', '_51DEGREES_LICENSE_KEY' - $license = [Environment]::GetEnvironmentVariable($licenseName) - if ($license) { + if ($licenseName -eq '_51DEGREES_LICENSE_KEY_51DID' -and $didLicense) { Write-Host "License key: $licenseName" - $env:_51DEGREES_LICENSE_KEY = $license + $env:_51DEGREES_LICENSE_KEY_51DID = $didLicense } else { Write-Host 'License key: none' - Remove-Item Env:_51DEGREES_LICENSE_KEY -ErrorAction SilentlyContinue + Remove-Item Env:_51DEGREES_LICENSE_KEY_51DID -ErrorAction SilentlyContinue } # Only the Pipeline API tree is built, as in its own repository. - mvn -B -f pipeline/pom.xml -pl pipeline.did -am test ` - -Dtest=DidClientLiveTests -DfailIfNoTests=false + mvn -B -f pipeline/pom.xml ` + -pl pipeline.did,pipeline.cloudrequestengine -am test ` + '-Dtest=DidClientLiveTests,CloudRequestEngineLiveTests#build_Process_LicenseKeyAlone*' ` + -DfailIfNoTests=false if ($LASTEXITCODE -ne 0) { $failed = $true } Write-Host '::endgroup::' } diff --git a/.github/workflows/data-file-change.yml b/.github/workflows/data-file-change.yml index 8fadf2a5b..22eaa8f3d 100644 --- a/.github/workflows/data-file-change.yml +++ b/.github/workflows/data-file-change.yml @@ -23,7 +23,4 @@ jobs: org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - data-key: ${{ secrets.DEVICE_DETECTION_KEY }} - data-url: ${{ secrets.DEVICE_DETECTION_URL }} + secrets: inherit diff --git a/.github/workflows/example-assets-update.yml b/.github/workflows/example-assets-update.yml index aff060ff8..0499b12f0 100644 --- a/.github/workflows/example-assets-update.yml +++ b/.github/workflows/example-assets-update.yml @@ -22,5 +22,4 @@ jobs: with: repo-name: ${{ github.event.repository.name }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} + secrets: inherit diff --git a/.github/workflows/examples-fresh-clone.yml b/.github/workflows/examples-fresh-clone.yml new file mode 100644 index 000000000..1eb6ca652 --- /dev/null +++ b/.github/workflows/examples-fresh-clone.yml @@ -0,0 +1,145 @@ +name: Examples from a fresh clone + +# Builds each product's console examples as AGENTS.md says, from a clone +# with its submodules, against the packages published on Maven Central, with +# the JDK and Maven the runner has. Then runs the getting started example +# from the repository root with no settings. A fresh clone holds only a link +# for each free data file. On Ubuntu the example downloads its file, which +# tests the download, and the job keeps the file in the cache. The other +# systems take it from there, so each file is downloaded once a run. The +# Ubuntu job also builds every example module, with its tests compiled, +# against the published packages. + +on: + pull_request: + paths: + - 'examples/**' + - 'cxx' + - '.github/workflows/examples-fresh-clone.yml' + - 'ci/run-fresh-clone-example.ps1' + push: + branches: + - main + - rc + schedule: + - cron: '30 4 * * *' + workflow_dispatch: + +permissions: + contents: read + +jobs: + download: + name: ${{ matrix.product }} on ubuntu-latest, downloading its data file + strategy: + fail-fast: false + matrix: + include: + - product: device-detection + example: fiftyone.devicedetection.examples.console.GettingStartedOnPrem + file: cxx/data/device-detection/51Degrees-LiteV4.1.hash + expect: '' + - product: ip-intelligence + example: fiftyone.ipintelligence.examples.console.GettingStartedOnPrem + file: cxx/data/ip-intelligence/51Degrees-LiteV41.ipi + # Each IP Intelligence value is printed with its weighting. + expect: '(weighting ' + runs-on: ubuntu-latest + steps: + - name: Clone the repository with its submodules + uses: actions/checkout@v4 + with: + submodules: recursive + persist-credentials: false + + - name: The runner's Java and Maven + shell: pwsh + run: | + java -version + mvn -v + + - name: Build the console examples against the published packages + working-directory: examples/${{ matrix.product }} + shell: pwsh + run: mvn -B --no-transfer-progress -pl console -am package -DskipTests + + - name: Run the getting started example from the repository root + shell: pwsh + run: >- + ./ci/run-fresh-clone-example.ps1 -Product '${{ matrix.product }}' + -Example '${{ matrix.example }}' -ExpectDownload + -Expect '${{ matrix.expect }}' + + # The key follows the link, so a new free file gets a new entry, and + # the same file is kept once rather than once a run. + - name: Keep the data file for the other systems + uses: actions/cache/save@v4 + with: + path: ${{ matrix.file }} + key: fresh-clone-${{ matrix.product }}-${{ hashFiles(format('{0}.link', matrix.file)) }} + enableCrossOsArchive: true + + # The console build above compiles the console examples and their + # tests. This builds every example module, the web examples too, with + # its tests compiled, against the published packages, so a test that + # uses a class the published packages lack fails here. It comes after + # the cache is saved, so the other systems still get the data file. The + # tests are not run here, as the pull request build runs them. + - name: Build every example and its tests against the published packages + working-directory: examples/${{ matrix.product }} + shell: pwsh + run: mvn -B --no-transfer-progress package -DskipTests + + other-systems: + name: ${{ matrix.product }} on ${{ matrix.os }} + needs: download + strategy: + fail-fast: false + matrix: + # macOS runners are shared by the whole organisation, so pull + # requests leave them out and the pushes and the daily run use them. + os: ${{ fromJSON(github.event_name == 'pull_request' && '["windows-latest"]' || '["windows-latest", "macos-latest"]') }} + product: [device-detection, ip-intelligence] + include: + - product: device-detection + example: fiftyone.devicedetection.examples.console.GettingStartedOnPrem + file: cxx/data/device-detection/51Degrees-LiteV4.1.hash + expect: '' + - product: ip-intelligence + example: fiftyone.ipintelligence.examples.console.GettingStartedOnPrem + file: cxx/data/ip-intelligence/51Degrees-LiteV41.ipi + # Each IP Intelligence value is printed with its weighting. + expect: '(weighting ' + runs-on: ${{ matrix.os }} + steps: + - name: Clone the repository with its submodules + uses: actions/checkout@v4 + with: + submodules: recursive + persist-credentials: false + + - name: Take the data file the Ubuntu job downloaded + uses: actions/cache/restore@v4 + with: + path: ${{ matrix.file }} + key: fresh-clone-${{ matrix.product }}-${{ hashFiles(format('{0}.link', matrix.file)) }} + enableCrossOsArchive: true + fail-on-cache-miss: true + + - name: The runner's Java and Maven + shell: pwsh + run: | + java -version + mvn -v + + - name: Build the console examples against the published packages + working-directory: examples/${{ matrix.product }} + shell: pwsh + run: mvn -B --no-transfer-progress -pl console -am package -DskipTests + + - name: Run the getting started example from the repository root + shell: pwsh + run: >- + ./ci/run-fresh-clone-example.ps1 -Product '${{ matrix.product }}' + -Example '${{ matrix.example }}' -ExpectDownload:$false + -Expect '${{ matrix.expect }}' diff --git a/.github/workflows/monthly-copyright-update.yml b/.github/workflows/monthly-copyright-update.yml index fdc6d3383..dde0e98af 100644 --- a/.github/workflows/monthly-copyright-update.yml +++ b/.github/workflows/monthly-copyright-update.yml @@ -17,5 +17,4 @@ jobs: repo-name: ${{ github.event.repository.name }} org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} + secrets: inherit diff --git a/.github/workflows/nightly-pipeline.yml b/.github/workflows/nightly-pipeline.yml index ce793b713..7b197d431 100644 --- a/.github/workflows/nightly-pipeline.yml +++ b/.github/workflows/nightly-pipeline.yml @@ -13,6 +13,19 @@ on: - cron: '0 3 * * *' jobs: + # The publish builds main, which the pull requests into rc do not change, so + # it runs first and has the runners before the pull request builds do. + Publish: + name: Nightly Publish + uses: 51Degrees/common-ci/.github/workflows/nightly-publish.yml@main + with: + repo-name: ${{ github.event.repository.name }} + org-name: ${{ github.event.repository.owner.login }} + dryrun: ${{ inputs.dryrun || false }} + common-ci-ref: ${{ inputs.common-ci-ref || '' }} + cache-assets: true + secrets: inherit + DataFileChange: name: Nightly Data File Change uses: 51Degrees/common-ci/.github/workflows/nightly-data-file-change.yml@main @@ -21,10 +34,7 @@ jobs: org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - data-key: ${{ secrets.DEVICE_DETECTION_KEY }} - data-url: ${{ secrets.DEVICE_DETECTION_URL }} + secrets: inherit PackageUpdate: name: Nightly Package Update @@ -34,8 +44,7 @@ jobs: org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} + secrets: inherit SubmoduleUpdate: name: Nightly Submodule Update @@ -45,12 +54,11 @@ jobs: org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} + secrets: inherit PullRequests: if: ${{ !cancelled() }} - needs: [DataFileChange, PackageUpdate, SubmoduleUpdate] + needs: [Publish, DataFileChange, PackageUpdate, SubmoduleUpdate] name: Nightly Pull Requests uses: 51Degrees/common-ci/.github/workflows/nightly-pull-requests.yml@main with: @@ -59,39 +67,6 @@ jobs: dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} cache-assets: true - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - - Publish: - if: ${{ !cancelled() }} - needs: PullRequests - name: Nightly Publish - uses: 51Degrees/common-ci/.github/workflows/nightly-publish.yml@main - with: - repo-name: ${{ github.event.repository.name }} - org-name: ${{ github.event.repository.owner.login }} - dryrun: ${{ inputs.dryrun || false }} - common-ci-ref: ${{ inputs.common-ci-ref || '' }} - cache-assets: true - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - MavenSettings: ${{ secrets.MAVEN_SETTINGS }} - JavaGpgKeyPassphrase: ${{ secrets.JAVA_GPG_KEY_PASSPHRASE }} - JavaPGP: ${{ secrets.JAVA_KEY_PGP_FILE }} - CodeSigningKeyVaultName: ${{ secrets.CODE_SIGNING_KEY_VAULT_NAME }} - CodeSigningKeyVaultUrl: ${{ secrets.CODE_SIGNING_KEY_VAULT_URL }} - CodeSigningKeyVaultClientId: ${{ secrets.CODE_SIGNING_KEY_VAULT_CLIENT_ID }} - CodeSigningKeyVaultTenantId: ${{ secrets.CODE_SIGNING_KEY_VAULT_TENANT_ID }} - CodeSigningKeyVaultClientSecret: ${{ secrets.CODE_SIGNING_KEY_VAULT_CLIENT_SECRET }} - CodeSigningKeyVaultCertificateName: ${{ secrets.CODE_SIGNING_KEY_VAULT_CERTIFICATE_NAME }} - CodeSigningKeyVaultCertificateData: ${{ secrets.CODE_SIGNING_KEY_VAULT_CERTIFICATE_DATA }} + # One configuration that fails does not cancel the others. + fail-fast: false + secrets: inherit diff --git a/.github/workflows/package-update.yml b/.github/workflows/package-update.yml index 37cd28ee4..3dacbc3fe 100644 --- a/.github/workflows/package-update.yml +++ b/.github/workflows/package-update.yml @@ -19,5 +19,4 @@ jobs: org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} + secrets: inherit diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7989697c2..9e0290371 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,20 +20,4 @@ jobs: dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} cache-assets: true - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - MavenSettings: ${{ secrets.MAVEN_SETTINGS }} - JavaGpgKeyPassphrase: ${{ secrets.JAVA_GPG_KEY_PASSPHRASE }} - JavaPGP: ${{ secrets.JAVA_KEY_PGP_FILE }} - CodeSigningKeyVaultName: ${{ secrets.CODE_SIGNING_KEY_VAULT_NAME }} - CodeSigningKeyVaultUrl: ${{ secrets.CODE_SIGNING_KEY_VAULT_URL }} - CodeSigningKeyVaultClientId: ${{ secrets.CODE_SIGNING_KEY_VAULT_CLIENT_ID }} - CodeSigningKeyVaultTenantId: ${{ secrets.CODE_SIGNING_KEY_VAULT_TENANT_ID }} - CodeSigningKeyVaultClientSecret: ${{ secrets.CODE_SIGNING_KEY_VAULT_CLIENT_SECRET }} - CodeSigningKeyVaultCertificateName: ${{ secrets.CODE_SIGNING_KEY_VAULT_CERTIFICATE_NAME }} - CodeSigningKeyVaultCertificateData: ${{ secrets.CODE_SIGNING_KEY_VAULT_CERTIFICATE_DATA }} + secrets: inherit diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index c1fda910b..a198b8b68 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -9,6 +9,10 @@ on: common-ci-ref: type: string default: '' + full-matrix: + description: 'Build every configuration, not only those the changes need' + type: boolean + default: false jobs: PullRequests: @@ -20,10 +24,10 @@ jobs: dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} cache-assets: true - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} + # Each pull request is built on the configurations its changes need, + # as ci/select-options.ps1 chooses, and one that fails does not cancel + # the others. With nothing eligible, the nightly builds rc instead. + full-matrix: ${{ inputs.full-matrix || false }} + fail-fast: false + build-without-pull-requests: false + secrets: inherit diff --git a/.github/workflows/release-pull-request.yml b/.github/workflows/release-pull-request.yml new file mode 100644 index 000000000..8e4af9491 --- /dev/null +++ b/.github/workflows/release-pull-request.yml @@ -0,0 +1,25 @@ +name: Release Pull Request + +# Builds and tests the release pull request, rc into main, on every entry in +# ci/options.json, and never merges it. A pull request into rc is built only on +# the entries its changes need, so this is the full build of a release before +# it reaches main. See common-ci .github/workflows/release-pull-request.yml. +on: + pull_request: + branches: [main] + +# A push to rc while the release is open builds it again, in place of the +# build of the older rc. +concurrency: + group: release-pull-request-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + Release: + name: Release Pull Request + uses: 51Degrees/common-ci/.github/workflows/release-pull-request.yml@main + with: + repo-name: ${{ github.event.repository.name }} + org-name: ${{ github.event.repository.owner.login }} + cache-assets: true + secrets: inherit diff --git a/.github/workflows/root-locale.yml b/.github/workflows/root-locale.yml new file mode 100644 index 000000000..ac8954fc9 --- /dev/null +++ b/.github/workflows/root-locale.yml @@ -0,0 +1,30 @@ +name: Root locale + +on: + pull_request: + push: + branches: + - main + - rc + workflow_dispatch: + +permissions: + contents: read + +jobs: + root-locale: + name: Every case change in the main code names its locale + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Test the check + shell: pwsh + run: Invoke-Pester -Path ./ci/check-root-locale.Tests.ps1 -CI + + - name: Check the case changes + shell: pwsh + run: ./ci/check-root-locale.ps1 diff --git a/.github/workflows/submodule-update.yml b/.github/workflows/submodule-update.yml index 7bf3291c8..d3b64cce2 100644 --- a/.github/workflows/submodule-update.yml +++ b/.github/workflows/submodule-update.yml @@ -19,5 +19,4 @@ jobs: org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} + secrets: inherit diff --git a/.gitmodules b/.gitmodules index f3fd5d50e..c3cdc8753 100644 --- a/.gitmodules +++ b/.gitmodules @@ -11,3 +11,6 @@ [submodule "pipeline/pipeline.core/configuration-validation"] path = pipeline/pipeline.core/configuration-validation url = ../configuration-validation +[submodule "pipeline/derived-properties"] + path = pipeline/derived-properties + url = ../derived-properties diff --git a/AGENTS.md b/AGENTS.md index b5125dfc6..aa823815a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,9 @@ the native engines from `cxx` and does need them, as the README says. ### Prerequisites - Git. -- A JDK, 8 or later. CI tests 8, 11, 17 and 21 (`ci/options.json`). +- A JDK, 8 or later. CI tests 8, 11, 17 and 21 (`ci/options.json`), and + builds and runs the examples on the JDK each runner has (see + [What CI tests](#pitfalls)). - Maven 3.8 or later, or the `mvnw` wrapper at the root. - Network access to GitHub, to Maven Central for the packages the examples use, and to `distributor.51degrees.com` for the data files. @@ -104,29 +106,33 @@ where `` is the `` in that examples folder's `pom.xml`, ### Run -Run the examples from the repository root. They find `logback.xml`, the data -file and their options file with `FileFinder`, which searches only below the -folder Java runs in (`pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/FileFinder.java`). -When a free data file is not there, the example finds its `.link` the same way -and runs the download script for it (`findDataFile` in each examples folder's -`shared` module, `DataFileHelper.java`), with Windows PowerShell on Windows and -`sh` on Linux and macOS. The same commands work on Windows, Linux and macOS. +Run the examples from the repository root, or from a product's examples +folder as its README does. They look for the data repository, `cxx/data`, in +the folder Java runs in and then in each folder above it, and find a data file +or its `.link` there (`findDataFile` in each examples folder's `shared` +module, `DataFileHelper.java`). When a free data file is not there, the +example runs the download script for its `.link`, with Windows PowerShell on +Windows and `sh` on Linux and macOS. They find `logback.xml` with +`FileFinder`, which searches below the folder Java runs in +(`pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/FileFinder.java`), +and the root and both examples folders have one. The same commands work on +Windows, Linux and macOS. ``` java -cp examples/device-detection/console/target/device-detection-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar fiftyone.devicedetection.examples.console.GettingStartedOnPrem ``` -The IP Intelligence example looks for the paid Enterprise file, -`ip-intelligence/51Degrees-EnterpriseIpiV41.ipi`, so give it the Lite file as -its first argument. +The IP Intelligence example uses the free Lite file, +`ip-intelligence/51Degrees-LiteV41.ipi`, when it is given no argument, and +downloads it first when it is not there. ``` -java -cp examples/ip-intelligence/console/target/ip-intelligence-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar fiftyone.ipintelligence.examples.console.GettingStartedOnPrem cxx/data/ip-intelligence/51Degrees-LiteV41.ipi +java -cp examples/ip-intelligence/console/target/ip-intelligence-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar fiftyone.ipintelligence.examples.console.GettingStartedOnPrem ``` -The example downloads the Lite file first when it is not there. In place of -the argument, the `_51DEGREES_IPI_PATH` environment variable or system property -can give the absolute path of a data file that is already there. +An absolute path as the first argument names another data file. Otherwise the +`_51DEGREES_IPI_PATH` environment variable or system property, when it names +a file that is there, comes before the argument or the Lite file. On Java 24 or later, add `--enable-native-access=ALL-UNNAMED` before the class name, as @@ -141,50 +147,57 @@ and the script's `Downloading` and `Saved` lines follow. The Device Detection example processes five sets of evidence. For each it prints `Input values:` and the evidence, then `Results:` with -`Mobile Device`, `Platform Name`, `Platform Version`, `Browser Name` and -`Browser Version`. Logback lines come before and after. The last ones report -the data file and end the run, with these messages. - -``` -Used a 'Lite' data file, created , days ago, from location '' -This example is using the 'Lite ' data file. This is used for illustration, ... -All done -``` - -A warning that the data file is more than 28 days old may come too, which is -expected with the Lite file. The exit code is 0. +`Mobile Device`, `Platform Name`, `Platform Version`, `Browser Name`, +`Browser Version`, `Hardware Vendor`, `Hardware Name` and `Device Type`. The +Lite file does not have the last three, so each of them reads +`needs the Enterprise data file`. The IP Intelligence example looks up `116.154.188.222`, `45.236.48.61` and an IPv6 address. For each it prints `Input values:` with `query.client-ip`, then `Results:` with a line for each property, from `RegisteredName` to -`TimeZoneOffset`. The Lite file has RegisteredCountry, RegisteredName and -RegisteredOwner, so a line for a property it does not have reads -`Unknown. The property '' is not available in this data file.` followed -by a link, and the first such line also logs a warning that some properties -need a paid subscription. It then logs the data file in the same way, and -`All done`. +`TimeZoneOffset`. Each value is followed by its weighting, as in +`RegisteredCountry: CN (weighting 1.00)`. The Lite file has RegisteredName, +RegisteredOwner and RegisteredCountry, so every other line reads `needs the +Enterprise data file`. + +Both then end the same way, where `` is `Lite` for Device Detection and +`IPIV4Lite` for IP Intelligence. -The paid data files have many more properties, as the -[pricing page](https://51degrees.com/pricing?utm_source=github&utm_medium=docs&utm_campaign=51degrees-java&utm_content=agents.md&utm_term=correct-output) -describes. +``` +Some of these properties need the Enterprise data file. Contact us for a copy at +INFO ... Used a '' data file, created , days ago, from location '' +INFO ... All done +``` + +Then comes the "Find out more" section, between two rules, with the +product's pages on 51degrees.com and the repositories. A warning that the +data file is more than 28 days old may come before `All done`, which is +expected with the Lite file. The exit code is 0. ### Cloud examples -The cloud examples need a resource key, which a person creates in the -configurator, so ask for one if none is set. A key with the properties most of -the Device Detection examples use can be made -[here](https://configure.51degrees.com/jqz435Nc?utm_source=github&utm_medium=docs&utm_campaign=51degrees-java&utm_content=agents.md&utm_term=cloud-examples). -The Device Detection cloud example takes the key as its first argument, or -from a `TestResourceKey` environment variable or system property. +The cloud examples need a resource key, which a person creates on the +configuration page, so ask for one if none is set. A key with the properties +most of the Device Detection examples use can be made +[here](https://51degrees.com/configure/jqz435Nc?utm_source=github&utm_medium=docs&utm_campaign=51degrees-java&utm_content=agents.md&utm_term=cloud-examples). +The Device Detection, IP Intelligence and Location cloud examples take the +key as their first argument, or from the `_51DEGREES_RESOURCE_KEY` +environment variable or system property, and no other name. The TAC and +native model examples need a key with paid properties and read it from +`_51DEGREES_RESOURCE_KEY_SUPER`, which each names when it has no key. ``` java -cp examples/device-detection/console/target/device-detection-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar fiftyone.devicedetection.examples.console.GettingStartedCloud ``` It prints the same `Input values:` and `Results:` blocks as the on-premise -example. The IP Intelligence cloud examples read their key from -`_51DEGREES_RESOURCE_KEY` first and `TestResourceKey` second, and -`examples/ip-intelligence/README.md` gives the keys to create for them. +example. `examples/ip-intelligence/README.md` gives the keys to create for +the IP Intelligence examples. + +With no key, a cloud example does not fail. It says that it needs a resource +key, names the variable to set, gives the page where a key is created, ends +with its "Find out more" section and exits with code 0. The tests named +`WithoutKey` run each one that way. ### Pitfalls @@ -196,16 +209,14 @@ example. The IP Intelligence cloud examples read their key from 'device-detection/51Degrees-LiteV4.1.hash' which must be somewhere in the project space to be found.` Run `git submodule update --init --recursive`. A zip download of the repository has no submodules, so clone with Git. -- **Running from another folder.** `FileFinder` looks only below the folder - Java runs in, so `java` started in `examples/device-detection`, as the - examples' README shows, cannot find the data file or its link in `cxx`. - Start it from the repository root. The same applies to `mvn exec:java`, +- **Running from outside the repository.** An example started in a folder + outside the repository, such as one the jar was copied to, finds no + `cxx/data` above it and no `logback.xml` below it. Start it in the + repository root or a folder inside it. The same applies to `mvn exec:java`, which runs in the folder Maven starts in. -- **The version in the jar name.** The examples' READMEs show jar names with - `4.4.20` and `4.4.19`, which are out of date. Use the name in `target`. -- **The IP Intelligence example without an argument.** It searches for the - Enterprise file, logs `Failed to find IP Intelligence data file` and stops, - even when the Lite file is in place. +- **The version in the jar name.** A jar carries the examples folder's own + ``, `4.5.127-SNAPSHOT` on `rc` today, not the version of the + packages the examples use, `4.5.130`. Use the name in `target`. - **The `dev` profile.** `-Pdev`, which CI uses, builds the examples against the libraries here, which needs `mvn install` at the root first and so the native build, with CMake 3.10 or later and a C++ toolchain. It is not needed @@ -215,25 +226,58 @@ example. The IP Intelligence cloud examples read their key from for Windows x64 and x86, Linux x64 and aarch64, and macOS x64 and aarch64. The `ip-intelligence.engine.on-premise` 4.5.128 jar has the same, without Windows x86. On a platform a jar lacks, the example cannot load the engine. -- **What CI does not test.** On a pull request CI builds and tests the - examples with the `dev` profile (`ci/build-project.ps1` and - `ci/run-unit-tests.ps1`), and against the published packages only when it - publishes (`ci/run-integration-tests.ps1`). It never runs the console - example jars as above, and the example tests use the paid data files when - CI has them. +- **What CI tests.** On a pull request CI builds and tests the examples + with the `dev` profile (`ci/build-project.ps1` and + `ci/run-unit-tests.ps1`), and the example tests use the paid data files + when CI has them. Publish tests them against the packages it has just + built (`ci/run-integration-tests.ps1`). + [examples-fresh-clone.yml](.github/workflows/examples-fresh-clone.yml) + does what [Run the examples](#run-the-examples) says, with the JDK and + Maven each runner has. It + builds the console examples against the published packages and runs each + product's `GettingStartedOnPrem` jar from the repository root with no + settings. On Ubuntu the free data file is downloaded, as from a fresh + clone, and the Windows and macOS jobs take that file from the run's cache, + so each file is downloaded once a run. The Ubuntu job then builds every + example module, the web examples too, with its tests compiled against + the published packages, so a test that uses a class only this + repository has fails there. It runs on each push to `main` and + `rc`, daily, and on a pull request that changes the examples. Its check + runs by hand from the repository root after the build, with + `-ExpectDownload` only when the data file is still a link: + `pwsh -File ./ci/run-fresh-clone-example.ps1 -Product device-detection -Example fiftyone.devicedetection.examples.console.GettingStartedOnPrem -ExpectDownload`. + +## Environment variable names + +[common-ci's list of environment variable names](https://github.com/51Degrees/common-ci/blob/d8a9c596eabf69dd5581a08e9360cdd1c9b3d9e8/environment-variables.md) +holds every environment variable name that the packages, tests, examples and +CI scripts of every language read, with what each name holds and the +deprecated names each replaced. The list decides. Use the names it gives, and +when a change needs a new name, or reads an old name again, change +[`environment-variables.json`](https://github.com/51Degrees/common-ci/blob/d8a9c596eabf69dd5581a08e9360cdd1c9b3d9e8/environment-variables.json) +in common-ci in the same piece of work. This repository's own name +check, [`ci/check-environment-variable-names.ps1`](ci/check-environment-variable-names.ps1), still keeps a +list of its own. Where the two differ, common-ci's list is the one to +follow. ## One reactor, six package trees The root `pom.xml` lists the six package trees as modules but is not their parent, so nothing from it reaches the published poms. Each tree keeps its own -parent pom. They all carry the same version, and the libraries reference each -other through `${project.version}`, so Maven resolves those dependencies from -the reactor. The examples reference published versions unless the `dev` -profile is active. Keep it that way: a reader of the examples should see the -package versions a project of their own would use. - -Because the root is not a parent, `versions:set` has to run in each tree. CI -does that. When changing versions by hand, run it in each of the six folders. +parent pom. In the repository they all carry the same development version, +and the libraries reference each other through `${project.version}`, so Maven +resolves those dependencies from the reactor. The examples reference published +versions unless the `dev` profile is active. Keep it that way, so that a +reader of the examples sees the package versions a project of their own would +use. + +Each published module is released at its own version, tagged +`/`. The publish writes that version into the module's +`pom.xml`, with the versions of its parent and of the packages it depends on, +so leave the versions in the poms as they are. When a module is added, or one +module starts to depend on another, add it to `ci/packages.json` as well, or +the pull request build fails. `ci/README.md` describes how a release is +planned. ## Data files and `project.root` @@ -275,6 +319,14 @@ The Windows pre-build scripts take a toolset and SDK version from the pom. JavaScript templates and the OWID library are submodules inside `pipeline`, where the Pipeline API expects them. +`pipeline/derived-properties` is the +[derived-properties](https://github.com/51Degrees/derived-properties) +repository. `pipeline.derivedproperty` packages the scripts in its `scripts` +folder as resources, so a script added there needs a `BuiltInScript` +constant, which `BuiltInScriptTests` checks. The package's tests run every +case file and every rejection case in its `tests` folder, so its answers and +its faults are checked against the other languages'. + ## Releases Pull requests go to `rc`, not `main`, and stay drafts until they have been @@ -287,3 +339,11 @@ into `main` from any other branch. Plain words, no em dashes, sentence case headings. Nothing in this public repository names a private repository or an issue in one. + +## Tests that fail on timing + +A test that fails on a time limit, a performance threshold or a timeout is +fixed, not run again until it passes. Either the limit is set to fit the +slowest runner the test runs on, with each runner's measurement behind it, or +the reason it takes too long is found and fixed. A failure that comes and goes +costs everyone who meets it the time to look into it. diff --git a/README.md b/README.md index df1cd6427..6e14ed6cf 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Each product is a separate Maven Central package. For Device Detection, cloud or The product guides below list the other packages. To build from source, see [Getting the code](#getting-the-code). -Try the [User-Agent tester](https://51degrees.com/developers/user-agent-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install) and [IP tester](https://51degrees.com/developers/ip-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install). For cloud use, [create a Resource Key](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install). +Try the [User-Agent tester](https://51degrees.com/developers/user-agent-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install) and [IP tester](https://51degrees.com/developers/ip-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install). For cloud use, [create a Resource Key](https://51degrees.com/configure/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=install). Created from `pipeline-java`, `device-detection-java`, `ip-intelligence-java`, `location-java`, `device-detection-java-examples`, and `ip-intelligence-java-examples`, this repository continues their work. @@ -98,10 +98,16 @@ mvn install -DskipTests -pl device-detection -am mvn surefire:test ``` -The cloud tests need a Resource Key, passed as `-DTestResourceKey=`. The +The cloud tests need a Resource Key, in the `_51DEGREES_RESOURCE_KEY` +environment variable, or passed as `-D_51DEGREES_RESOURCE_KEY=`, and are +skipped without one. The on-premise tests need the free data files, which the script in [Getting the code](#getting-the-code) downloads. For the Enterprise IP Intelligence data file, see `ip-intelligence/README.md`. + +Every environment variable name that the packages, tests, examples and CI +scripts read is in [common-ci's list of environment variable names](https://github.com/51Degrees/common-ci/blob/d8a9c596eabf69dd5581a08e9360cdd1c9b3d9e8/environment-variables.md), +which decides. Tests find data files by searching from the root of this repository, so run Maven from here rather than from inside a package folder. @@ -132,9 +138,12 @@ does the same. ## Versions -The repository has one version, and every package is released with it. CI -sets the version on each package tree before a release and tags the commit -with the plain version, such as `4.5.127`. +Each package has its own version, and a release publishes only the packages +that changed since their last release and the packages that depend on them. +A change only to documentation, tests, examples or the descriptive parts of a +`pom.xml`, such as its links, releases nothing. +Each is tagged with its name and version, such as `pipeline.core/4.5.131`. +[ci/README.md](ci/README.md) describes how CI works out what to release. ## History diff --git a/ci/README.md b/ci/README.md index b00a5034a..12070f0c0 100644 --- a/ci/README.md +++ b/ci/README.md @@ -11,15 +11,35 @@ repositories used to do for one package tree each. publish, the two on-premise engines that build a native library, and the generated accessors. The other scripts loop over it rather than naming folders themselves. -- The root `pom.xml` builds every tree in one reactor but is not their parent, - so [build-package.ps1](build-package.ps1) has common-ci set the version in - each tree before it deploys from the root. The examples are built and tested - but left out of the deploy. +- [packages.json](packages.json) lists every published module as a package, + named by its artifactId, with its folder, the other folders its build takes + files from, and the packages it depends on, its parent included. A parent's + folder is its `pom.xml` alone, because the rest of the folder is its + modules. [build-project.ps1](build-project.ps1) fails a pull request whose + poms refer to a package that `DependsOn` does not name, or add a module + that has no entry. +- The root `pom.xml` builds every tree in one reactor but is not their parent. + [build-package.ps1](build-package.ps1) writes each published module's own + version into its `pom.xml`, with the versions of its parent and of the + packages it depends on, and deploys only the modules being released. The + examples are built and tested but left out of the deploy. - [build-project.ps1](build-project.ps1) and [run-unit-tests.ps1](run-unit-tests.ps1) use the `dev` profile, so a pull request tests the examples against the libraries it changes. The publish workflow instead tests the examples against the version it built, through [run-integration-tests.ps1](run-integration-tests.ps1). +- [select-options.ps1](select-options.ps1) chooses the entries a pull request + runs. The standard set is JDK 17 and JDK 8 on Linux, without the three IP + Intelligence tests in [standard-excludes.txt](standard-excludes.txt) and + with the Selenium contract on JDK 17 alone. A change to a native module, IP + Intelligence, a `pom.xml` or a file outside the package trees adds to it. + macOS and performance run only when `rc` itself or the release pull request + is built, which + [release-pull-request.yml](../.github/workflows/release-pull-request.yml) + does on every entry. The Pester 5 tests of select-options.ps1 are + [select-options.Tests.ps1](select-options.Tests.ps1), which + [ci-script-tests.yml](../.github/workflows/ci-script-tests.yml) runs on + Linux and Windows for each pull request that changes this folder. - [fetch-assets.ps1](fetch-assets.ps1) fetches every data file as one of common-ci's assets, which each run caches once, and links or copies it into `cxx/data/device-detection` or `cxx/data/ip-intelligence`, where the tests @@ -28,13 +48,62 @@ repositories used to do for one package tree each. of cxx. No test job downloads a data file, so a file added to the data repository needs an entry in common-ci's `steps/fetch-assets.ps1` before CI can use it. +- [run-fresh-clone-example.ps1](run-fresh-clone-example.ps1) runs a + product's console example jar from the repository root for + [examples-fresh-clone.yml](../.github/workflows/examples-fresh-clone.yml). + It fails unless the example exits with 0 and prints results, and unless it + downloads its free data file exactly when the job expects it to. That + workflow's Ubuntu job downloads each product's free data file once a run, + as a test of the download, and its other jobs take the file from the cache + the Ubuntu job saves. To run it by hand, build the console examples as + [AGENTS.md](../AGENTS.md#build) says, then from the repository root give + `-ExpectDownload` when the free data file is only a link, and leave it out + when the file is there. + + ``` + pwsh -File ./ci/run-fresh-clone-example.ps1 -Product device-detection -Example fiftyone.devicedetection.examples.console.GettingStartedOnPrem -ExpectDownload + ``` + + [run-fresh-clone-example.Tests.ps1](run-fresh-clone-example.Tests.ps1) + tests it with `pwsh -File` and with the workflow's own calls. - [run-performance-tests.ps1](run-performance-tests.ps1) measures the Pipeline API, Device Detection and IP Intelligence and writes one results file per configuration. The Device Detection and IP Intelligence metrics are prefixed with the product, as both use the same names. -- The repository has one version, tagged as a plain version such as `4.5.127`. - [get-next-package-version.ps1](get-next-package-version.ps1) moves only the - patch number. +- Each package has its own version, tagged `/`, such as + `pipeline.core/4.5.131`. A publish releases a package when a file under its + folders changed since its last tag, and every package that depends on it, + each at its last tag with the patch number moved. Changes only to Markdown, + `LICENSE` files, or files under folders named `docs`, `tests`, `test` or + `examples`, which takes in each module's `src/test`, release nothing, and + [release-plan-filter.ps1](release-plan-filter.ps1) leaves out a `pom.xml` + whose only change is to its `name`, `description`, `url`, `licenses`, + `developers`, `scm` or `issueManagement`, or to its comments and layout. + Its Pester 5 tests are + [release-plan-filter.Tests.ps1](release-plan-filter.Tests.ps1), which + [ci-script-tests.yml](../.github/workflows/ci-script-tests.yml) runs on + Linux and Windows for each pull request that changes this folder. + [release.ps1](release.ps1) takes that plan from common-ci's + `steps/release-plan.ps1`, and the publish scripts build, check and tag from + it. [check-version-bump.ps1](check-version-bump.ps1) and + [check-release-interval.ps1](check-release-interval.ps1) check each package + against its own tags, for a major or minor move and for a release less than + a week after its last. +- [publish-package.ps1](publish-package.ps1) uploads the one publishing + bundle, then pushes every package's tag in one atomic push, so a publish + that stops leaves every package tagged or none, and then makes one GitHub + release per tag. Maven Central refuses a version it already has, so when + the push fails after the upload, push the tags the log names by hand + rather than running the publish again. When `gh release create` fails + part way, every tag is pushed but the packages after the failure have no + GitHub release, and a publish run again plans nothing and so makes none. + Make each missing release by hand. For each tag the log names after + "Pushing" for which `gh release view ` finds no release, run + `gh release create --verify-tag --title " " + --notes " : ."`, with the reason the Configure + job's plan gives for that package, as the script does. Its Pester 5 + tests are [publish-package.Tests.ps1](publish-package.Tests.ps1), which the + same workflow runs. ## Secrets @@ -44,23 +113,33 @@ Required: [access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#about-personal-access-tokens) for cloning repositories and creating pull requests. +The workflows pass the secrets to common-ci with `secrets: inherit`, and +common-ci exports each under its own name before the scripts here run. The +tests read only these names. + To run the on-premise tests: -- `DEVICE_DETECTION_KEY`: [license key](https://51degrees.com/pricing?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=ci-readme.md&utm_term=secrets) - for the Device Detection TAC data file and the data file update example. -- `IP_INTELLIGENCE_KEY`: license key for the IP Intelligence Enterprise data - file. -- `IPI_DATA_FILE_URL`: URL of the IP Intelligence Enterprise data file. The - IP Intelligence examples and benchmark are skipped without it. -- `DEVICE_DETECTION_URL` (optional): URL of the Device Detection TAC data file. +- `_51DEGREES_LICENSE_KEY`: [license key](https://51degrees.com/pricing?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=ci-readme.md&utm_term=secrets) + for the data files. common-ci downloads the Device Detection TAC and the IP + Intelligence Enterprise data files from the Distributor with it, and the + data file update example uses it. To run the cloud tests: - `_51DEGREES_RESOURCE_KEY_BESPOKE`: [resource key](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=ci-readme.md&utm_term=secrets) - passed to the tests as `TestResourceKey` and `SuperResourceKey`. -- `_51DEGREES_LICENSE_KEY_51DID`: license key for the 51Did cloud tests, - together with the `_51DEGREES_RESOURCE_KEY_BASIC` and - `_51DEGREES_RESOURCE_KEY_51DID` variables. + from which common-ci fills `_51DEGREES_RESOURCE_KEY`, the key the cloud + tests read, and `_51DEGREES_RESOURCE_KEY_SUPER`, the key with every + property, which the Location and the TAC and native model tests read. +- `_51DEGREES_LICENSE_KEY_BESPOKE`: license key for the cloud products, for + the Cloud Request Engine's test on a license key alone. +- `_51DEGREES_RESOURCE_KEY_51DID`: resource key for the 51Did cloud tests. + It has to belong to the same account as `_51DEGREES_LICENSE_KEY_51DID`, + or the licensed redeem comes back unreadable. +- `_51DEGREES_LICENSE_KEY_51DID`: license key for the 51Did cloud tests. + `cloud-51did.yml` gives it only to the pass with + `_51DEGREES_RESOURCE_KEY_51DID`. The Bespoke pass redeems with none. + +A cloud test with no key is skipped, naming the variable it reads. To publish to Maven Central (51Degrees only): diff --git a/ci/build-package.ps1 b/ci/build-package.ps1 index 75464f09f..99df8cfa3 100644 --- a/ci/build-package.ps1 +++ b/ci/build-package.ps1 @@ -7,7 +7,15 @@ param( $ErrorActionPreference = "Stop" $PSNativeCommandUseErrorActionPreference = $true -. "$PSScriptRoot/packages.ps1" +. "$PSScriptRoot/release.ps1" + +# Builds the packages this publish releases, each at its own version from the +# release plan (see release.ps1). $Version is common-ci's label for the run and +# is not used. +$plan = @(Get-BuildPlan $RepoName) +Write-Host "Building $($plan.Count) packages:" +$plan | Format-Table Name, Previous, Version, Reason -AutoSize | Out-String -Width 200 | + Write-Host # Put the native libraries built on every platform into the engine they came # from, so each engine's jar carries all of its own and none of the other's. @@ -21,30 +29,25 @@ foreach ($package in $NativePackages) { } } -# The examples are never published. They are left out of the deploy so that -# the last project in the reactor is one the publishing plugin is configured -# for. Maven 3 does not exclude a folder's modules with the folder, so each of -# them is named. -$repoPath = (Resolve-Path $RepoName).Path -$examples = foreach ($package in $Packages | Where-Object { -not $_.Publish }) { - Get-ChildItem -Path "$RepoName/$($package.Path)" -Recurse -Filter pom.xml | - Where-Object { $_.FullName -notmatch '[\\/](target|node_modules)[\\/]' } | - ForEach-Object { '!' + [IO.Path]::GetRelativePath($repoPath, $_.DirectoryName).Replace('\', '/') } -} +# Every published module gets its own version, and the versions of its parent +# and of the packages it depends on, including those this publish does not +# release, which stay at their last release. +Set-PomVersions $RepoName (Get-BuildVersions $RepoName $plan) + +# Only the modules in the plan are deployed, so the one publishing bundle holds +# only them. A package they depend on that is not released is taken from Maven +# Central at its last release, the version their poms now require. +$modules = @($plan | ForEach-Object { Get-ModulePath $_ }) -# common-ci sets the version, deploys from the root to a local staging folder -# with signing, and copies the result and the publishing bundle into the -# package folder. The root pom.xml is not the parent of the package trees, so -# setting the version at the root would not reach them. VersionPaths names each -# tree, the examples included, so the whole repository carries the version -# being released. The publishing plugin writes the one bundle for the whole -# reactor under the first tree that publishes, and common-ci finds it there. +# common-ci deploys from the root to a local staging folder with signing, and +# copies the result and the publishing bundle into the package folder. The +# versions are already set, so it is given no folder to set them in. ./java/build-package.ps1 ` -RepoName $RepoName ` -Name $Name ` -Version $Version ` - -VersionPaths $Packages.Path ` - -ExtraArgs "-DskipNativeBuild=true", "-pl", ($examples -join ',') ` + -VersionPaths @() ` + -ExtraArgs "-DskipNativeBuild=true", "-pl", ($modules -join ',') ` -JavaGpgKeyPassphrase $Keys['JavaGpgKeyPassphrase'] ` -JavaPGP $Keys['JavaPGP'] ` -CodeSigningKeyVaultName: $Keys['CodeSigningKeyVaultName'] ` @@ -54,5 +57,18 @@ $examples = foreach ($package in $Packages | Where-Object { -not $_.Publish }) { -CodeSigningKeyVaultClientSecret $Keys['CodeSigningKeyVaultClientSecret'] ` -CodeSigningKeyVaultCertificateName $Keys['CodeSigningKeyVaultCertificateName'] ` -CodeSigningKeyVaultCertificateData $Keys['CodeSigningKeyVaultCertificateData'] +$exitCode = $LASTEXITCODE + +# Maven Central limits each organisation's release size and file count in a +# calendar month, so the log gives both for the bundle this publish uploads. +$bundle = Get-Item package/central-bundle.zip +$zip = [IO.Compression.ZipFile]::OpenRead($bundle.FullName) +try { + $files = $zip.Entries.Count +} finally { + $zip.Dispose() +} +Write-Host ("The publishing bundle for $($plan.Count) packages is " + + "$($bundle.Length) bytes ({0:N1} MiB) in $files files" -f ($bundle.Length / 1MB)) -exit $LASTEXITCODE +exit $exitCode diff --git a/ci/build-project.ps1 b/ci/build-project.ps1 index d8524aa3e..eea288f3f 100644 --- a/ci/build-project.ps1 +++ b/ci/build-project.ps1 @@ -9,6 +9,14 @@ $PSNativeCommandUseErrorActionPreference = $true Write-Host "JAVA_HOME: $ENV:JAVA_HOME" +# A publish releases a package's dependents only when packages.json names +# them, so a module or dependency it leaves out fails here, before the change +# is merged, rather than at the publish. +$gaps = @(Get-PackagesJsonGaps $RepoName) +if ($gaps.Count -gt 0) { + throw "ci/packages.json does not match the poms:`n$($gaps -join "`n")" +} + # One build of the whole reactor, native libraries included. The dev profile # builds the examples against the libraries here rather than the versions # published to Maven Central, so a change to a library is tested by the diff --git a/ci/check-environment-variable-names.Tests.ps1 b/ci/check-environment-variable-names.Tests.ps1 index 84460b9a8..348ba0190 100644 --- a/ci/check-environment-variable-names.Tests.ps1 +++ b/ci/check-environment-variable-names.Tests.ps1 @@ -49,6 +49,30 @@ Describe 'Find-EnvironmentNameProblem' { Get-Kind $Line | Should -Contain 'fallback' } + # The names this repository's tests, examples and CI scripts used for + # keys and the cloud address before each had a _51DEGREES_ name, read as + # a Java string, a system property passed with -D, a PowerShell + # parameter or variable, a placeholder in a configuration file, or a key + # in a hashtable. + It 'finds the old key name ' -ForEach @( + @{ Line = '.setResourceKey(TestKeyUtils.getNamedKey("TestResourceKey"))' } + @{ Line = 'public static final String RESOURCE_KEY_ENV_VAR = "SuperResourceKey";' } + @{ Line = 'UPDATE_EXAMPLE_LICENSE_KEY_NAME = "LicenseKey";' } + @{ Line = ' "TestLicenseKey",' } + @{ Line = ' "_51DEGREES_LICENSE_KEY_CLOUDV5BESPOKE",' } + @{ Line = 'CLOUD_END_POINT_NAME = "TestCloudEndPoint";' } + @{ Line = '${TestCloudEndpoint:-https://cloud.51degrees.com/api/v4}' } + @{ Line = '${SuperResourceKey}' } + @{ Line = '${AcceptChNoneKey}' } + @{ Line = ' "-DTestResourceKey=$TestResourceKey"' } + @{ Line = ' "-DLicenseKey=$DeviceDetection"' } + @{ Line = ' [string]$TestResourceKey,' } + @{ Line = ' TestResourceKey = $TestResourceKey' } + @{ Line = 'mvn clean test -DTestResourceKey=[Resource Key]' } + ) { + Get-Kind $Line | Should -Contain 'fallback' + } + # The cloud request engine's tests name the query string parameter # resource_key, which is on the wire and not in the environment, so the # old names keep their case. @@ -65,6 +89,12 @@ Describe 'Find-EnvironmentNameProblem' { @{ Line = ' PAID_RESOURCE_KEY: ${{ env.PAID_RESOURCE_KEY }}' } @{ Line = 'System.getenv("SUPER_RESOURCE_KEY")' } @{ Line = 'Set the key with !!YOUR_RESOURCE_KEY!! in the readme.' } + @{ Line = ' * https://distributor.51degrees.com/api/v2/download?LicenseKeys=' } + @{ Line = ' .setDataUpdateLicenseKey(licenseKey)' } + @{ Line = ' * method for a configuration option named ''LicenseKey''. If the' } + @{ Line = ' "_51DEGREES_LICENSE_KEY_BESPOKE",' } + @{ Line = ' resourceKey = KeyHelper.getOrSetTestResourceKey(resourceKey);' } + @{ Line = '${_51DEGREES_RESOURCE_KEY_SUPER}' } ) { Get-Kind $Line | Should -BeNullOrEmpty } @@ -90,7 +120,7 @@ Describe 'the check over a repository' { } } - It 'passes an old name in a file that reads it as a deprecated fallback' { + It 'passes an old name in a file that keeps a deprecated constant holding it' { $repo = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid()) New-Item -ItemType Directory $repo | Out-Null try { @@ -124,14 +154,13 @@ Describe 'the check over a repository' { Describe 'Test-DeprecatedFallback' { - # An old name is allowed only in a file that reads it as a deprecated - # fallback or tests that read, and only the names listed for that file. + # An old name is allowed only in a file that keeps a deprecated constant + # holding it or tests that constant, and only the names listed for that + # file. It 'allows in ' -ForEach @( @{ Path = 'pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java'; Value = '"FOD_CLOUD_API_URL' } - @{ Path = 'pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java'; Value = '"RESOURCE_KEY' } - @{ Path = 'pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java'; Value = '"LICENSE_KEY' } @{ Path = 'ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java'; Value = '51DEGREES_IPI_PATH' } - @{ Path = 'examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml'; Value = '51DEGREES_DD_PATH' } + @{ Path = 'examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java'; Value = '51DEGREES_RESOURCE_KEY' } ) { Test-DeprecatedFallback $Path $Value | Should -BeTrue } @@ -144,8 +173,122 @@ Describe 'Test-DeprecatedFallback' { @{ Path = 'pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java'; Value = '"FOD_CLOUD_API_URL' } @{ Path = 'ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java'; Value = '51DEGREES_DD_PATH' } @{ Path = 'ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java'; Value = '51degrees_ipi_path' } - @{ Path = 'examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml'; Value = '51DEGREES_IPI_PATH' } + @{ Path = 'examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml'; Value = '51DEGREES_DD_PATH' } + @{ Path = 'pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java'; Value = '"RESOURCE_KEY' } ) { Test-DeprecatedFallback $Path $Value | Should -BeFalse } } + +Describe 'the workflows' { + + # The 51Did license key unseals only results made with the resource key + # of its own account. Filled from the Bespoke key, the licensed redeem + # comes back unreadable, so the 51Did resource key is taken from the + # secret of its own name. + It 'fill _51DEGREES_RESOURCE_KEY_51DID from the secret of the same name' { + $workflows = Join-Path $PSScriptRoot '../.github/workflows' + $lines = @(Get-ChildItem $workflows -Filter *.yml | + Select-String -Pattern '^\s*_51DEGREES_RESOURCE_KEY_51DID\s*:' | + ForEach-Object { "$($_.Filename):$($_.LineNumber): $($_.Line.Trim())" }) + $lines | Should -Not -BeNullOrEmpty + $lines | Where-Object { + $_ -notmatch ':\s*\$\{\{\s*secrets\._51DEGREES_RESOURCE_KEY_51DID\s*\}\}\s*$' + } | Should -BeNullOrEmpty + } + + # cloud-51did.yml runs the 51Did tests once for each resource key. The + # tests redeem with _51DEGREES_LICENSE_KEY_51DID, which unseals only + # results made with _51DEGREES_RESOURCE_KEY_51DID. So each pass gets only + # the license whose suffix matches its resource key, and the Bespoke pass + # gets none. The step's script runs here with mvn replaced by a function + # that records the keys each pass hands it. + Context 'cloud-51did.yml' { + + BeforeAll { + $workflow = Join-Path $PSScriptRoot '../.github/workflows/cloud-51did.yml' + $lines = @(Get-Content $workflow) + $start = $lines | Select-String -Pattern '^(\s*)run: \|\s*$' | + Select-Object -First 1 + $indent = $start.Matches[0].Groups[1].Value.Length + $body = foreach ($line in $lines[$start.LineNumber..($lines.Count - 1)]) { + if ($line.Trim() -and $line -notmatch "^\s{$($indent + 1),}") { break } + if ($line.Length -gt $indent + 2) { $line.Substring($indent + 2) } else { '' } + } + $script:Step = [scriptblock]::Create($body -join "`n") + } + + BeforeEach { + $script:Saved = @(Get-ChildItem env: | + Where-Object { $_.Name -like '_51DEGREES_*' }) + $script:Saved | ForEach-Object { Remove-Item "Env:$($_.Name)" } + $env:_51DEGREES_RESOURCE_KEY_51DID = 'resource-51did' + $env:_51DEGREES_RESOURCE_KEY_BESPOKE = 'resource-bespoke' + $env:_51DEGREES_LICENSE_KEY_51DID = 'license-51did' + $env:_51DEGREES_LICENSE_KEY_BESPOKE = 'license-bespoke' + } + + AfterEach { + Get-ChildItem env: | Where-Object { $_.Name -like '_51DEGREES_*' } | + ForEach-Object { Remove-Item "Env:$($_.Name)" } + $script:Saved | ForEach-Object { Set-Item "Env:$($_.Name)" $_.Value } + } + + It 'redeems with the 51Did license only in the pass with the 51Did resource key' { + $passes = [System.Collections.Generic.List[object]]::new() + function mvn { + $passes.Add([pscustomobject]@{ + Resource = $env:_51DEGREES_RESOURCE_KEY + License = $env:_51DEGREES_LICENSE_KEY_51DID + }) + $global:LASTEXITCODE = 0 + } + & $script:Step 6> $null + + @($passes.Resource | Sort-Object) | + Should -Be @('resource-51did', 'resource-bespoke') + ($passes | Where-Object Resource -eq 'resource-51did').License | + Should -Be 'license-51did' + ($passes | Where-Object Resource -eq 'resource-bespoke').License | + Should -BeNullOrEmpty + } + + # A license key with no resource key of its own suffix means a secret + # is missing. The licensed redeem would then never run, and the job + # would still pass, so the step fails naming the missing secret. + It 'fails naming when has a value and it is empty' -ForEach @( + @{ Resource = '_51DEGREES_RESOURCE_KEY_51DID'; License = '_51DEGREES_LICENSE_KEY_51DID' } + @{ Resource = '_51DEGREES_RESOURCE_KEY_BESPOKE'; License = '_51DEGREES_LICENSE_KEY_BESPOKE' } + ) { + $passes = [System.Collections.Generic.List[object]]::new() + function mvn { + $passes.Add($env:_51DEGREES_RESOURCE_KEY) + $global:LASTEXITCODE = 0 + } + Remove-Item "Env:$Resource" + + { & $script:Step 6> $null } | Should -Throw "*$Resource*" + $passes | Should -BeNullOrEmpty + } + + # A deployment without 51Did has neither secret, and the runner then + # sets both names empty. The Bespoke pass must still run, with no + # 51Did license and no failure. + It 'runs the Bespoke pass alone when both 51Did keys are empty' { + $passes = [System.Collections.Generic.List[object]]::new() + function mvn { + $passes.Add([pscustomobject]@{ + Resource = $env:_51DEGREES_RESOURCE_KEY + License = $env:_51DEGREES_LICENSE_KEY_51DID + }) + $global:LASTEXITCODE = 0 + } + $env:_51DEGREES_RESOURCE_KEY_51DID = '' + $env:_51DEGREES_LICENSE_KEY_51DID = '' + + { & $script:Step 6> $null } | Should -Not -Throw + @($passes.Resource) | Should -Be @('resource-bespoke') + $passes[0].License | Should -BeNullOrEmpty + } + } +} diff --git a/ci/check-environment-variable-names.ps1 b/ci/check-environment-variable-names.ps1 index 11d75e85c..33cfadf1d 100644 --- a/ci/check-environment-variable-names.ps1 +++ b/ci/check-environment-variable-names.ps1 @@ -1,11 +1,11 @@ # Every 51Degrees environment variable name in this repository starts with an # underscore, because a POSIX shell cannot set a name that starts with a digit, -# and each variable has one name. An old name is still read, as a deprecated -# fallback for when its new name is not set, by the files listed in -# DeprecatedFallbacks below and by nothing else. This reads every file git -# tracks, which leaves out build output and the submodules, and fails on any -# name that starts with the digit and on any old name, outside the files -# listed for that name. +# and each variable has one name. No old name is read. An old name may appear +# only in the files listed in KeptOldNames below, which keep a deprecated +# public constant holding it, and in the tests of those constants. This reads +# every file git tracks, which leaves out build output and the submodules, and +# fails on any name that starts with the digit and on any old name, outside +# the files listed for that name. # # Dot-sourcing the script defines Find-EnvironmentNameProblem and # Test-DeprecatedFallback without reading the repository, which is how its @@ -51,30 +51,38 @@ $script:DigitInitialPattern = [regex]::new( # query string parameter is on the wire rather than in the environment, so # matching in any case would fail on a name that is not an environment # variable at all. +# +# The second part holds the names this repository used for its keys and for +# the cloud address before each had a _51DEGREES_ name, and the dropped +# _51DEGREES_LICENSE_KEY_CLOUDV5BESPOKE. A test read them as a Java string, +# Maven passed them as system properties with -D, a configuration file +# named them as placeholders, and the CI scripts took them as parameters +# and set them as hashtable keys, so a dollar and -D come before one as +# well. A single quote does not, because the pipeline's documentation names +# a builder option 'LicenseKey', which is not an environment variable. $script:FallbackPattern = [regex]::new( '(?:["'']|(?i:env)[:.]\\?|\$\{|getenv\(\s*["'']|^[\t\- ]*)' + '(?:RESOURCE_KEY|LICENSE_KEY|FOD_CLOUD_API_URL)(?![A-Za-z0-9_])|' + + '(?:"|(?i:env)[:.]\\?|\$\{?|-D|getenv\(\s*"|^[\t\- ]*)' + + '(?:TestResourceKey|SuperResourceKey|TestLicenseKey|LicenseKey|' + + 'TestCloudEnd[Pp]oint|AcceptCh(?:None|Platform|Browser|Hardware)Key|' + + '_51DEGREES_LICENSE_KEY_CLOUDV5BESPOKE)(?![A-Za-z0-9_])|' + '\bLEGACY_[A-Z_]*ENV_VARS?\b') -# The files that read an old name as a deprecated fallback, or that test -# that read, each with the old names it may hold. The names are put together +# The files that keep a deprecated public constant holding an old name, or +# that test the constant, each with the old names it may hold. The names are put together # from parts so that this file does not match itself. $oldEndpoint = 'FOD_CLOUD' + '_API_URL' $oldResourceKey = 'RESOURCE' + '_KEY' -$oldLicenseKey = 'LICENSE' + '_KEY' $oldDigitKey = $digitInitial + $oldResourceKey $oldIpiPath = $digitInitial + 'IPI_PATH' -$oldDdPath = $digitInitial + 'DD_PATH' $cloudEngine = 'pipeline/pipeline.cloudrequestengine/src' $did = 'pipeline/pipeline.did/src' -$examples = 'pipeline/pipeline.developer-examples/pipeline.developer-examples' $ipiShared = 'ip-intelligence/ip-intelligence.shared/src' $ipiHelpers = 'fiftyone/ipintelligence/shared/testhelpers' $ipiExamples = 'examples/ip-intelligence/shared/src' $ipiExampleHelpers = 'fiftyone/ipintelligence/examples/shared' -$onPrem = 'examples/device-detection/web/getting-started.onprem/src' -$onPremPackage = 'fiftyone/devicedetection/examples/web' -$script:DeprecatedFallbacks = @{ +$script:KeptOldNames = @{ "$cloudEngine/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java" = @($oldEndpoint) "$cloudEngine/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilderTests.java" = @@ -83,18 +91,6 @@ $script:DeprecatedFallbacks = @{ @($oldEndpoint) "$did/test/java/fiftyone/pipeline/did/DidClientEnvironmentTests.java" = @($oldEndpoint) - "$did/test/java/fiftyone/pipeline/did/LiveCloudKeys.java" = - @($oldResourceKey, $oldLicenseKey) - "$did/test/java/fiftyone/pipeline/did/LiveCloudKeysTests.java" = - @($oldResourceKey, $oldLicenseKey) - "$examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java" = - @($oldEndpoint, $oldResourceKey) - "$examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/EndpointTests.java" = - @($oldEndpoint, $oldResourceKey) - "$examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java" = - @($oldResourceKey, $oldLicenseKey) - "$examples.fodid/src/test/java/pipeline/developerexamples/fodid/DeprecatedKeyNameTests.java" = - @($oldResourceKey, $oldLicenseKey) "$ipiShared/main/java/$ipiHelpers/FileUtils.java" = @($oldIpiPath) "$ipiShared/test/java/$ipiHelpers/FileUtilsTest.java" = @($oldIpiPath) "$ipiShared/main/java/$ipiHelpers/KeyUtils.java" = @($oldDigitKey) @@ -107,21 +103,13 @@ $script:DeprecatedFallbacks = @{ @($oldDigitKey) "$ipiExamples/test/java/$ipiExampleHelpers/KeyHelperTest.java" = @($oldDigitKey) - 'ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java' = - @($oldIpiPath) - "$onPrem/main/webapp/WEB-INF/51Degrees-OnPrem.xml" = @($oldDdPath) - "$onPrem/main/java/$onPremPackage/GettingStartedWebOnPrem.java" = - @($oldDdPath) - "$onPrem/test/java/$onPremPackage/OnPremDataFileOptionTest.java" = - @($oldDdPath) } - # Whether the text matched in the file at this path, relative to the root # with forward slashes, is an old name that file is listed for. The name is # the run of name characters the match ends with, compared in its exact # case. function Test-DeprecatedFallback([string]$Path, [string]$Value) { - $allowed = $script:DeprecatedFallbacks[$Path] + $allowed = $script:KeptOldNames[$Path] if ($null -eq $allowed) { return $false } diff --git a/ci/check-release-interval.ps1 b/ci/check-release-interval.ps1 new file mode 100644 index 000000000..305e15512 --- /dev/null +++ b/ci/check-release-interval.ps1 @@ -0,0 +1,42 @@ +param ( + [Parameter(Mandatory)][string]$RepoName, + [string]$Repository, + [string]$HowToAllow +) +$ErrorActionPreference = "Stop" +. "$PSScriptRoot/release.ps1" + +# The shared publish runs this in place of its own check, because each package +# has its own version and tags (see release.ps1). Every package about to be +# published goes through the shared steps/check-release-interval.ps1, run from +# the common-ci root, against its own '/' tags, so each +# package gets at most one new version a week. That refuses a package +# released less than a week ago unless ALLOW_EARLY_RELEASE is 'true', from the +# date in WEEKLY_RELEASES_FROM. + +$plan = @(Get-ReleasePlan $RepoName) +if ($plan.Count -eq 0) { + Write-Host "No package is to be published, so there is nothing to check." + exit 0 +} + +$arguments = @{ RepoName = $RepoName; Repository = $Repository } +if ($HowToAllow) { + $arguments.HowToAllow = $HowToAllow +} + +$refused = @() +foreach ($release in $plan) { + Write-Host "$($release.Name):" + $global:LASTEXITCODE = 0 + ./steps/check-release-interval.ps1 @arguments -TagPrefix $release.Name + if ($LASTEXITCODE -ne 0) { + $refused += $release.Name + } +} + +if ($refused) { + Write-Host "Nothing is released, because these packages were released less than a week ago: $($refused -join ', ')." + exit 1 +} +exit 0 diff --git a/ci/check-root-locale.Tests.ps1 b/ci/check-root-locale.Tests.ps1 new file mode 100644 index 000000000..1d1292be8 --- /dev/null +++ b/ci/check-root-locale.Tests.ps1 @@ -0,0 +1,98 @@ +# Tests for ci/check-root-locale.ps1. + +BeforeAll { + $script:Check = Join-Path $PSScriptRoot 'check-root-locale.ps1' + . $script:Check +} + +Describe 'Find-DefaultLocaleCaseChange' { + + It 'finds ' -ForEach @( + @{ Line = 'String key = property.getName().toLowerCase();' } + @{ Line = 'if (name.toUpperCase().equals(other)) {' } + @{ Line = 'key.toLowerCase( ).contains("x")' } + ) { + Find-DefaultLocaleCaseChange -Path 'A.java' -Text $Line | + Should -Be "A.java:1: $Line" + } + + It 'leaves alone' -ForEach @( + @{ Line = 'String key = property.getName().toLowerCase(Locale.ROOT);' } + @{ Line = 'name.toUpperCase(Locale.ENGLISH)' } + @{ Line = 'a.equalsIgnoreCase(b)' } + @{ Line = 'Character.toLowerCase(c)' } + ) { + Find-DefaultLocaleCaseChange -Path 'A.java' -Text $Line | + Should -BeNullOrEmpty + } + + It 'gives each line number' { + $text = "class A {`n String a = b.toLowerCase();`r`n String c = d;`n" + + " String e = f.toUpperCase();`n}" + Find-DefaultLocaleCaseChange -Path 'p/A.java' -Text $text | + Should -Be @( + 'p/A.java:2: String a = b.toLowerCase();' + 'p/A.java:4: String e = f.toUpperCase();') + } + + It 'finds nothing in an empty file' { + Find-DefaultLocaleCaseChange -Path 'A.java' -Text '' | + Should -BeNullOrEmpty + } +} + +Describe 'Test-MainSource' { + + It 'reads ' -ForEach @( + @{ Path = 'pipeline/pipeline.core/src/main/java/a/B.java' } + @{ Path = 'src/main/java/B.java' } + @{ Path = 'examples/device-detection/shared/src/main/java/a/B.java' } + ) { + Test-MainSource $Path | Should -BeTrue + } + + It 'leaves out ' -ForEach @( + @{ Path = 'pipeline/pipeline.core/src/test/java/a/B.java' } + @{ Path = 'pipeline/pipeline.core/src/main/resources/a.json' } + @{ Path = 'ci/check-root-locale.ps1' } + @{ Path = 'docs/src/main.java.md' } + ) { + Test-MainSource $Path | Should -BeFalse + } +} + +Describe 'The check' { + + BeforeAll { + $script:Repo = Join-Path ([IO.Path]::GetTempPath()) ([guid]::NewGuid()) + New-Item -ItemType Directory -Path $script:Repo | Out-Null + & git -C $script:Repo init -q + $main = Join-Path $script:Repo 'm/src/main/java' + $test = Join-Path $script:Repo 'm/src/test/java' + New-Item -ItemType Directory -Path $main, $test | Out-Null + Set-Content -LiteralPath (Join-Path $test 'T.java') ` + -Value 'class T { String a = b.toLowerCase(); }' + Set-Content -LiteralPath (Join-Path $main 'M.java') ` + -Value 'class M { String a = b.toLowerCase(Locale.ROOT); }' + & git -C $script:Repo add -A + } + + AfterAll { + Remove-Item -LiteralPath $script:Repo -Recurse -Force + } + + It 'passes when every main case change names its locale' { + $output = & pwsh -NoProfile -File $script:Check -RepoRoot $script:Repo + $LASTEXITCODE | Should -Be 0 + ($output -join "`n") | Should -Match 'Every case change in 1 main' + } + + It 'fails on a main case change with no locale, naming the line' { + Set-Content -LiteralPath (Join-Path $script:Repo 'm/src/main/java/N.java') ` + -Value "class N {`n String a = b.toUpperCase();`n}" + & git -C $script:Repo add -A + $output = & pwsh -NoProfile -File $script:Check -RepoRoot $script:Repo + $LASTEXITCODE | Should -Be 1 + $output | Should -Contain 'm/src/main/java/N.java:2: String a = b.toUpperCase();' + } +} diff --git a/ci/check-root-locale.ps1 b/ci/check-root-locale.ps1 new file mode 100644 index 000000000..2713cc958 --- /dev/null +++ b/ci/check-root-locale.ps1 @@ -0,0 +1,67 @@ +# Every case change in the main code names its locale. Java's toLowerCase() +# and toUpperCase() with no argument use the default locale, and under a +# Turkish or Azeri default a capital I lowers to a dotless i, so a name that +# holds one no longer matches its lower case form. This reads every Java file +# git tracks under a src/main folder, which leaves out the tests, build output +# and the submodules, and fails on any call that names no locale. Pass +# Locale.ROOT. +# +# Dot-sourcing the script defines Find-DefaultLocaleCaseChange and +# Test-MainSource without reading the repository, which is how its tests use +# them. + +[CmdletBinding()] +param( + # The root of the checkout to read. The default is the repository this + # script lives in. + [string]$RepoRoot = (Join-Path $PSScriptRoot '..') +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$script:DefaultLocalePattern = [regex]::new('\.to(?:Lower|Upper)Case\(\s*\)') + +function Find-DefaultLocaleCaseChange { + # One ":: " for each line of the text with a case + # change that names no locale. + param( + [Parameter(Mandatory)][string]$Path, + [Parameter(Mandatory)][AllowEmptyString()][string]$Text + ) + $lines = $Text -split "`r?`n" + for ($i = 0; $i -lt $lines.Count; $i++) { + if ($script:DefaultLocalePattern.IsMatch($lines[$i])) { + "${Path}:$($i + 1): $($lines[$i].Trim())" + } + } +} + +function Test-MainSource { + # Whether a path as git gives it is a Java file under a src/main folder. + param([Parameter(Mandatory)][string]$Path) + $Path.EndsWith('.java') -and "/$Path".Contains('/src/main/') +} + +if ($MyInvocation.InvocationName -eq '.') { + return +} + +$root = (Resolve-Path $RepoRoot).Path +$tracked = & git -C $root -c core.quotePath=false ls-files +if ($LASTEXITCODE -ne 0) { + throw "git ls-files failed in $root" +} +$files = @($tracked | Where-Object { Test-MainSource $_ }) +$found = @(foreach ($file in $files) { + $text = [string](Get-Content -Raw -LiteralPath (Join-Path $root $file)) + Find-DefaultLocaleCaseChange -Path $file -Text $text +}) +if ($found.Count -gt 0) { + $found | ForEach-Object { Write-Output $_ } + Write-Output ("$($found.Count) case changes name no locale. Pass " + + 'Locale.ROOT, because the default locale can lower a capital I ' + + 'to a dotless i.') + exit 1 +} +Write-Output "Every case change in $($files.Count) main source files names its locale." diff --git a/ci/check-version-bump.ps1 b/ci/check-version-bump.ps1 new file mode 100644 index 000000000..f53fae22d --- /dev/null +++ b/ci/check-version-bump.ps1 @@ -0,0 +1,36 @@ +param ( + [Parameter(Mandatory)][string]$RepoName, + [string]$Version +) +$ErrorActionPreference = "Stop" +. "$PSScriptRoot/release.ps1" + +# The shared publish runs this in place of its own check, because each package +# has its own version (see release.ps1). Every package about to be published +# goes through the shared steps/check-version-bump.ps1, run from the common-ci +# root, against its own '/' tags. That refuses a major or +# minor move unless ALLOW_VERSION_BUMP is 'true'. $Version is common-ci's label +# for the run. + +Write-Host "The publish worked out $Version, its highest. Each package is checked on its own." + +$plan = @(Get-ReleasePlan $RepoName) +if ($plan.Count -eq 0) { + Write-Host "No package is to be published, so there is nothing to check." + exit 0 +} + +$refused = @() +foreach ($release in $plan) { + $global:LASTEXITCODE = 0 + ./steps/check-version-bump.ps1 -Version $release.Version -RepoName $RepoName -TagPrefix $release.Name + if ($LASTEXITCODE -ne 0) { + $refused += $release.Name + } +} + +if ($refused) { + Write-Host "Nothing is published, because these packages were refused: $($refused -join ', ')." + exit 1 +} +exit 0 diff --git a/ci/fetch-assets.ps1 b/ci/fetch-assets.ps1 index 600d3f6fc..790c4c7b8 100644 --- a/ci/fetch-assets.ps1 +++ b/ci/fetch-assets.ps1 @@ -1,11 +1,9 @@ -param ( - [string]$DeviceDetection, - [string]$DeviceDetectionUrl, - [string]$IpIntelligence, - [string]$IpIntelligenceUrl -) $ErrorActionPreference = "Stop" +# common-ci's step downloads the TAC and Enterprise data files from the +# Distributor with the license key in _51DEGREES_LICENSE_KEY, which it +# exports before this runs. + # Tests and examples find data files by searching below the repository root, # so every asset is linked into the folders of the data repository, which is # the data submodule of cxx. @@ -14,12 +12,12 @@ $deviceDetectionData = "$data/device-detection" $ipIntelligenceData = "$data/ip-intelligence" $deviceDetectionAssets = 'TAC-HashV41.hash', '51Degrees-LiteV4.1.hash', '20000 Evidence Records.yml', '20000 User Agents.csv' -./steps/fetch-assets.ps1 -DeviceDetection:$DeviceDetection -DeviceDetectionUrl:$DeviceDetectionUrl -Assets $deviceDetectionAssets +./steps/fetch-assets.ps1 -Assets $deviceDetectionAssets foreach ($asset in $deviceDetectionAssets) { New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/$asset" -Path "$deviceDetectionData/$asset" } -./steps/fetch-assets.ps1 -IpIntelligence:$IpIntelligence -IpIntelligenceUrl:$IpIntelligenceUrl -Assets '51Degrees-EnterpriseIpiV41.ipi', '51Degrees-LiteIpiV41.ipi', '51Degrees-IPIV4AsnIpiV41.ipi' +./steps/fetch-assets.ps1 -Assets '51Degrees-EnterpriseIpiV41.ipi', '51Degrees-LiteIpiV41.ipi', '51Degrees-IPIV4AsnIpiV41.ipi' New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/51Degrees-EnterpriseIpiV41.ipi" -Path "$ipIntelligenceData/51Degrees-EnterpriseIpiV41.ipi" # The asset is fetched under the blob name, but the tests, DataFileHelper and # the data repository scripts use the unpacked name 51Degrees-LiteV41.ipi, so diff --git a/ci/get-next-package-version.ps1 b/ci/get-next-package-version.ps1 index de0e6c291..ddfc48d64 100644 --- a/ci/get-next-package-version.ps1 +++ b/ci/get-next-package-version.ps1 @@ -3,31 +3,21 @@ param ( [Parameter(Mandatory)][string]$VariableName ) $ErrorActionPreference = "Stop" +. "$PSScriptRoot/release.ps1" -# The whole repository has one version, tagged as a plain version such as -# 4.5.127. Only the patch number moves here. The major and minor numbers -# change only when someone edits FirstVersion before the first release or -# pushes a tag by hand. - -# The version used while the repository has no version tag. It is above every -# version the absorbed repositories published, so the first release from here -# is an ordinary upgrade for each package. -$FirstVersion = '4.5.127' - -$tags = @(git -C $RepoName tag --list | - Where-Object { $_ -match '^\d+\.\d+\.\d+$' } | - Sort-Object { [version]$_ }) -$headTags = @(git -C $RepoName tag --points-at HEAD | - Where-Object { $_ -match '^\d+\.\d+\.\d+$' }) - -if ($headTags) { - # Already released, so the publish sees the tag and does nothing. - $next = $headTags | Sort-Object { [version]$_ } | Select-Object -Last 1 -} elseif ($tags) { - $highest = [version]$tags[-1] - $next = "$($highest.Major).$($highest.Minor).$($highest.Build + 1)" +# common-ci's publish carries one version through its jobs. This repository +# releases each package at its own version (see release.ps1), so that one +# version is only a label for the run, being the highest version this publish +# releases. Whether anything is released is decided by +# package-update-required.ps1, and each package's version and tag by +# build-package.ps1 and publish-package.ps1, all from the same release plan. +$plan = @(Get-ReleasePlan $RepoName) +if ($plan.Count -gt 0) { + $label = ($plan.Version | Sort-Object { [version]$_ })[-1] } else { - $next = $FirstVersion + # Nothing to release. Any valid version will do, because + # package-update-required.ps1 stops the publish before it is used. + $label = '4.5.0' } -Set-Variable -Scope Global -Name $VariableName -Value $next +Set-Variable -Scope Global -Name $VariableName -Value $label diff --git a/ci/package-update-required.ps1 b/ci/package-update-required.ps1 new file mode 100644 index 000000000..da231faad --- /dev/null +++ b/ci/package-update-required.ps1 @@ -0,0 +1,23 @@ +param ( + [Parameter(Mandatory)][string]$RepoName, + [string]$Version +) +$ErrorActionPreference = "Stop" +. "$PSScriptRoot/release.ps1" + +# Called by common-ci's publish in place of its own check, because a plain tag +# lookup cannot tell this repository's packages apart. Exits 0 when the release +# plan has something in it and 1 when it is empty, as common-ci's check does. +# $Version is common-ci's label for the run and is not used here. Each +# package's version is checked by check-version-bump.ps1. + +$plan = @(Get-ReleasePlan $RepoName) +if ($plan.Count -eq 0) { + Write-Output "No package changed since its last release, so there is nothing to publish" + exit 1 +} + +Write-Output "This publish releases $($plan.Count) packages, in this order:" +$plan | Format-Table Name, Previous, Version, Reason -AutoSize | Out-String -Width 200 | + Write-Output +exit 0 diff --git a/ci/packages.json b/ci/packages.json new file mode 100644 index 000000000..ea903985f --- /dev/null +++ b/ci/packages.json @@ -0,0 +1,49 @@ +[ + {"Name": "pipeline", "Path": "pipeline/pom.xml", "DependsOn": [], "Publish": true}, + {"Name": "pipeline.common", "Path": "pipeline/pipeline.common", "DependsOn": ["pipeline"], "Publish": true}, + {"Name": "pipeline.core", "Path": "pipeline/pipeline.core", "DependsOn": ["pipeline", "pipeline.common"], "Publish": true}, + {"Name": "pipeline.did", "Path": "pipeline/pipeline.did", "ExtraPaths": ["pipeline/owid-java"], "DependsOn": ["pipeline"], "Publish": true}, + {"Name": "pipeline.engines", "Path": "pipeline/pipeline.engines", "DependsOn": ["pipeline", "pipeline.core", "pipeline.caching", "pipeline.common"], "Publish": true}, + {"Name": "pipeline.caching", "Path": "pipeline/pipeline.caching", "DependsOn": ["pipeline", "pipeline.common"], "Publish": true}, + {"Name": "pipeline.cloudrequestengine", "Path": "pipeline/pipeline.cloudrequestengine", "DependsOn": ["pipeline", "pipeline.engines", "pipeline.common"], "Publish": true}, + {"Name": "pipeline.engines.fiftyone", "Path": "pipeline/pipeline.engines.fiftyone", "DependsOn": ["pipeline", "pipeline.engines", "pipeline.common"], "Publish": true}, + {"Name": "pipeline.translation", "Path": "pipeline/pipeline.translation", "DependsOn": ["pipeline", "pipeline.engines", "pipeline.core"], "Publish": true}, + {"Name": "pipeline.derivedproperty", "Path": "pipeline/pipeline.derivedproperty", "ExtraPaths": ["pipeline/derived-properties"], "DependsOn": ["pipeline", "pipeline.engines", "pipeline.core"], "Publish": true, "FirstVersion": "4.6.2"}, + {"Name": "pipeline.javascriptbuilder", "Path": "pipeline/pipeline.javascriptbuilder", "DependsOn": ["pipeline", "pipeline.jsonbuilder", "pipeline.engines", "pipeline.common"], "Publish": true}, + {"Name": "pipeline.jsonbuilder", "Path": "pipeline/pipeline.jsonbuilder", "DependsOn": ["pipeline", "pipeline.engines.fiftyone", "pipeline.common", "pipeline.engines"], "Publish": true}, + {"Name": "pipeline.web.packages", "Path": "pipeline/pipeline.web.packages/pom.xml", "DependsOn": ["pipeline"], "Publish": true}, + {"Name": "pipeline.web.shared", "Path": "pipeline/pipeline.web.packages/pipeline.web.shared", "DependsOn": ["pipeline.web.packages", "pipeline.javascriptbuilder"], "Publish": true}, + {"Name": "pipeline.web", "Path": "pipeline/pipeline.web.packages/pipeline.web", "DependsOn": ["pipeline.web.packages", "pipeline.web.shared", "pipeline.core"], "Publish": true}, + {"Name": "pipeline.web.mvc", "Path": "pipeline/pipeline.web.packages/pipeline.web.mvc", "DependsOn": ["pipeline.web.packages", "pipeline.web", "pipeline.core"], "Publish": true}, + {"Name": "pipeline-aggregate", "Path": "pipeline/pipeline.aggregate", "DependsOn": ["pipeline", "pipeline.web", "pipeline.javascriptbuilder", "pipeline.cloudrequestengine"], "Publish": true}, + {"Name": "pipeline.developer-examples", "Path": "pipeline/pipeline.developer-examples/pom.xml", "DependsOn": ["pipeline"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.web.shared", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.web.shared", "DependsOn": ["pipeline.developer-examples"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.flowelement", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement", "DependsOn": ["pipeline.developer-examples", "pipeline.core"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.onpremiseengine", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine", "DependsOn": ["pipeline.developer-examples", "pipeline.engines"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.cloudengine", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine", "DependsOn": ["pipeline.developer-examples", "pipeline.cloudrequestengine"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.clientsideelement", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element", "DependsOn": ["pipeline.developer-examples", "pipeline.engines"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.clientsideelement.mvc", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element-mvc", "DependsOn": ["pipeline.developer-examples", "pipeline.developerexamples.clientsideelement", "pipeline.web.mvc", "pipeline.developerexamples.web.shared"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.usagesharing", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing", "DependsOn": ["pipeline.developer-examples", "pipeline.engines.fiftyone"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.fodid", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid", "DependsOn": ["pipeline.developer-examples", "pipeline.did"], "Publish": true, "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.developerexamples.derivedproperty", "Path": "pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty", "DependsOn": ["pipeline.developer-examples", "pipeline.derivedproperty"], "Publish": true, "FirstVersion": "4.6.2", "Ignore": ["pipeline/pipeline.developer-examples/**"]}, + {"Name": "pipeline.device-detection", "Path": "device-detection/pom.xml", "DependsOn": ["pipeline-aggregate"], "Publish": true}, + {"Name": "device-detection.shared", "Path": "device-detection/device-detection.shared", "DependsOn": ["pipeline.device-detection", "pipeline.engines.fiftyone"], "Publish": true}, + {"Name": "device-detection.hash.engine.on-premise", "Path": "device-detection/device-detection.hash.engine.on-premise", "ExtraPaths": ["cxx"], "DependsOn": ["pipeline.device-detection", "device-detection.shared"], "Publish": true}, + {"Name": "device-detection.cloud", "Path": "device-detection/device-detection.cloud", "DependsOn": ["pipeline.device-detection", "device-detection.shared", "pipeline.cloudrequestengine"], "Publish": true}, + {"Name": "device-detection", "Path": "device-detection/device-detection", "DependsOn": ["pipeline.device-detection", "device-detection.hash.engine.on-premise", "device-detection.cloud", "pipeline-aggregate", "device-detection.shared"], "Publish": true}, + {"Name": "pipeline.ip-intelligence", "Path": "ip-intelligence/pom.xml", "DependsOn": ["pipeline-aggregate"], "Publish": true}, + {"Name": "ip-intelligence.shared", "Path": "ip-intelligence/ip-intelligence.shared", "DependsOn": ["pipeline.ip-intelligence", "pipeline.engines.fiftyone"], "Publish": true}, + {"Name": "ip-intelligence.translation", "Path": "ip-intelligence/ip-intelligence.translation", "ExtraPaths": ["cxx"], "DependsOn": ["pipeline.ip-intelligence", "ip-intelligence.shared", "pipeline.translation", "ip-intelligence.engine.on-premise"], "Publish": true}, + {"Name": "ip-intelligence.engine.on-premise", "Path": "ip-intelligence/ip-intelligence.engine.on-premise", "ExtraPaths": ["cxx"], "DependsOn": ["pipeline.ip-intelligence", "ip-intelligence.shared"], "Publish": true}, + {"Name": "ip-intelligence.cloud", "Path": "ip-intelligence/ip-intelligence.cloud", "DependsOn": ["pipeline.ip-intelligence", "ip-intelligence.shared", "pipeline.cloudrequestengine"], "Publish": true}, + {"Name": "ip-intelligence", "Path": "ip-intelligence/ip-intelligence", "DependsOn": ["pipeline.ip-intelligence", "ip-intelligence.engine.on-premise", "ip-intelligence.cloud", "pipeline-aggregate", "ip-intelligence.shared"], "Publish": true}, + {"Name": "pipeline.geo-location", "Path": "location/pom.xml", "DependsOn": [], "Publish": true}, + {"Name": "geo-location.core", "Path": "location/geo-location.core", "DependsOn": ["pipeline.geo-location", "pipeline.cloudrequestengine"], "Publish": true}, + {"Name": "geo-location.cloud", "Path": "location/geo-location.cloud", "DependsOn": ["pipeline.geo-location", "geo-location.core", "pipeline.cloudrequestengine", "pipeline.engines.fiftyone"], "Publish": true}, + {"Name": "geo-location", "Path": "location/geo-location", "DependsOn": ["pipeline.geo-location", "geo-location.cloud", "pipeline.cloudrequestengine", "pipeline.common"], "Publish": true}, + {"Name": "geo-location.examples", "Path": "location/geo-location.shell.examples", "DependsOn": ["pipeline.geo-location", "geo-location", "device-detection", "pipeline.engines"], "Publish": true, "Ignore": ["location/geo-location.shell.examples/**"]}, + {"Name": "geo-location.web.examples", "Path": "location/geo-location.web.examples/pom.xml", "DependsOn": ["pipeline.geo-location"], "Publish": true, "Ignore": ["location/geo-location.web.examples/**"]}, + {"Name": "geo-location.web.examples.mvc", "Path": "location/geo-location.web.examples/geo-location.web.examples.mvc", "DependsOn": ["geo-location.web.examples", "pipeline.web.mvc", "geo-location.cloud", "geo-location.web.examples.shared"], "Publish": true, "Ignore": ["location/geo-location.web.examples/**"]}, + {"Name": "geo-location.web.examples.servlet", "Path": "location/geo-location.web.examples/geo-location.web.examples.servlet", "DependsOn": ["geo-location.web.examples", "pipeline.web", "geo-location.cloud", "geo-location.web.examples.shared"], "Publish": true, "Ignore": ["location/geo-location.web.examples/**"]}, + {"Name": "geo-location.web.examples.shared", "Path": "location/geo-location.web.examples/geo-location.web.examples.shared", "DependsOn": ["geo-location.web.examples"], "Publish": true, "Ignore": ["location/geo-location.web.examples/**"]} +] diff --git a/ci/packages.ps1 b/ci/packages.ps1 index e1319727e..e3e153f2d 100644 --- a/ci/packages.ps1 +++ b/ci/packages.ps1 @@ -2,10 +2,12 @@ # relative to the repository root. # The package trees in this repository. The root pom.xml builds them all in -# one reactor but is not their parent, so a version change has to be made in -# each of them. Publish says whether the tree's packages go to Maven Central; -# the examples are built and tested but never published. -$Packages = @( +# one reactor but is not their parent. Publish says whether the tree's +# packages go to Maven Central. The examples are built and tested but never +# published. Each published module is a package of its own in packages.json. +# The example modules inside the published trees are packages too, and give +# an Ignore glob over their whole folder, so a change to one releases nothing. +$Trees = @( @{ Path = 'pipeline'; Publish = $true } @{ Path = 'device-detection'; Publish = $true } @{ Path = 'ip-intelligence'; Publish = $true } @@ -40,3 +42,134 @@ $Accessors = @( # Arguments every Maven call in these scripts passes. $MavenOptions = '--batch-mode', '--no-transfer-progress' + +$PomNamespace = 'http://maven.apache.org/POM/4.0.0' + +# The packages in packages.json that are published, one per Maven module, +# each named by its artifactId. +function Get-PublishedPackages([string]$RepoPath) { + return @(Get-Content -Raw (Join-Path $RepoPath 'ci/packages.json') | + ConvertFrom-Json | Where-Object { $_.Publish -eq $true }) +} + +# A parent's Path is its pom.xml alone, because its folder also holds its +# modules, whose changes are not changes to the parent. +function Get-PomPath($Package) { + if ($Package.Path.EndsWith('/pom.xml') -eq $true) { + return $Package.Path + } + return "$($Package.Path)/pom.xml" +} + +# The folder Maven builds a package in. +function Get-ModulePath($Package) { + return (Get-PomPath $Package) -replace '/pom\.xml$', '' +} + +# A pom.xml read with its whitespace kept, so that writing it back changes +# only what was set. +function Read-Pom([string]$Path) { + # .NET resolves a relative path from the process's folder, which is not + # the folder PowerShell is in after a Set-Location. + $Path = (Resolve-Path -LiteralPath $Path).ProviderPath + $xml = [Xml.XmlDocument]::new() + $xml.PreserveWhitespace = $true + $xml.Load($Path) + $namespaces = [Xml.XmlNamespaceManager]::new($xml.NameTable) + $namespaces.AddNamespace('m', $PomNamespace) + return [pscustomobject]@{ Path = $Path; Xml = $xml; Ns = $namespaces } +} + +function Save-Pom($Pom) { + $settings = [Xml.XmlWriterSettings]::new() + $settings.Encoding = [Text.UTF8Encoding]::new($false) + $settings.NewLineHandling = [Xml.NewLineHandling]::None + $writer = [Xml.XmlWriter]::Create($Pom.Path, $settings) + try { + $Pom.Xml.Save($writer) + } finally { + $writer.Dispose() + } +} + +function Get-PomText($Node, [string]$Name, $Ns) { + $child = $Node.SelectSingleNode("m:$Name", $Ns) + if ($null -eq $child) { + return $null + } + return $child.InnerText.Trim() +} + +# The other packages a pom.xml refers to, being its parent and its +# dependencies on them in any scope, wherever they are declared. Version is the element that +# holds the version, or $null where a parent pom manages it. +function Get-PomReferences($Pom, [string[]]$Names) { + $self = Get-PomText $Pom.Xml.DocumentElement 'artifactId' $Pom.Ns + $parent = $Pom.Xml.SelectSingleNode('/m:project/m:parent', $Pom.Ns) + if ($null -ne $parent) { + $name = Get-PomText $parent 'artifactId' $Pom.Ns + if ($Names -contains $name) { + [pscustomobject]@{ + Name = $name + Version = $parent.SelectSingleNode('m:version', $Pom.Ns) + } + } + } + foreach ($dependency in $Pom.Xml.SelectNodes('//m:dependency', $Pom.Ns)) { + $name = Get-PomText $dependency 'artifactId' $Pom.Ns + $group = Get-PomText $dependency 'groupId' $Pom.Ns + if ($Names -contains $name -and $name -ne $self -and + $group -in 'com.51degrees', '${project.groupId}') { + [pscustomobject]@{ + Name = $name + Version = $dependency.SelectSingleNode('m:version', $Pom.Ns) + } + } + } +} + +# The modules of the published trees, as folders, read from each pom's +# modules. +function Get-PublishedModules([string]$RepoPath) { + $folders = [Collections.Generic.Queue[string]]::new() + foreach ($tree in $Trees | Where-Object { $_.Publish -eq $true }) { + $folders.Enqueue($tree.Path) + } + while ($folders.Count -gt 0) { + $folder = $folders.Dequeue() + $folder + $pom = Read-Pom (Join-Path $RepoPath $folder 'pom.xml') + foreach ($module in $pom.Xml.SelectNodes('/m:project/m:modules/m:module', $pom.Ns)) { + $folders.Enqueue("$folder/$($module.InnerText.Trim())") + } + } +} + +# What packages.json leaves out, as one line each. Every published module must +# be a package, named by its artifactId, and each package's DependsOn must name +# every other package its pom.xml refers to. A dependency it leaves out would +# not be released when the package it depends on is, so its users would not +# get the fix. +function Get-PackagesJsonGaps([string]$RepoPath) { + $packages = Get-PublishedPackages $RepoPath + $names = @($packages.Name) + $folders = @($packages | ForEach-Object { Get-ModulePath $_ }) + foreach ($folder in Get-PublishedModules $RepoPath) { + if ($folders -notcontains $folder) { + "$folder/pom.xml is a published module with no entry in ci/packages.json" + } + } + foreach ($package in $packages) { + $pom = Read-Pom (Join-Path $RepoPath (Get-PomPath $package)) + $artifactId = Get-PomText $pom.Xml.DocumentElement 'artifactId' $pom.Ns + if ($artifactId -ne $package.Name) { + "$(Get-PomPath $package) is $artifactId, but ci/packages.json names it $($package.Name)" + } + $missing = @(Get-PomReferences $pom $names | + Where-Object { $package.DependsOn -notcontains $_.Name } | + ForEach-Object Name | Select-Object -Unique) + foreach ($name in $missing) { + "$(Get-PomPath $package) refers to $name, which the DependsOn of $($package.Name) in ci/packages.json does not name" + } + } +} diff --git a/ci/publish-package.Tests.ps1 b/ci/publish-package.Tests.ps1 new file mode 100644 index 000000000..5a4c177b0 --- /dev/null +++ b/ci/publish-package.Tests.ps1 @@ -0,0 +1,101 @@ +# Pester 5 tests of publish-package.ps1. The upload script is a stub, and git, +# gh and the release plan are mocked, so each test reads only what the publish +# would tag, push and release. + +BeforeAll { + $Script = Join-Path $PSScriptRoot 'publish-package.ps1' + + # release.ps1 needs common-ci's steps/release-plan.ps1, and the publish + # runs ./java/publish-package-maven.ps1 from the folder it is run in. + $Root = Join-Path ([IO.Path]::GetTempPath()) "publish-package-$PID" + $null = New-Item -ItemType Directory -Force ` + (Join-Path $Root 'common-ci' 'steps'), (Join-Path $Root 'java') + Set-Content (Join-Path $Root 'common-ci' 'steps' 'release-plan.ps1') ` + 'function Get-ReleasePlan([string]$RepoPath) { }' + Set-Content (Join-Path $Root 'java' 'publish-package-maven.ps1') ` + 'param($RepoName, $MavenSettings, $Version, $DryRun)' + $script:CommonCi = $env:COMMON_CI + $env:COMMON_CI = Join-Path $Root 'common-ci' + + # Pester mocks only a command it can find, so the plan is defined here + # as well as in the stub. + function Get-ReleasePlan([string]$RepoPath) { } + + function Invoke-Publish([bool]$DryRun = $false) { + Push-Location $Root + try { + & $Script -RepoName 'repo' -OrgName 'org' -Version '0' ` + -Keys @{ MavenSettings = '' } -DryRun $DryRun + } finally { + Pop-Location + } + } +} + +AfterAll { + $env:COMMON_CI = $script:CommonCi + Remove-Item -Recurse -Force $Root +} + +Describe 'publish-package.ps1' { + BeforeEach { + $global:SkipUpdateTag = $false + Mock git { } + Mock gh { } + Mock Get-ReleasePlan { + [pscustomobject]@{ + Name = 'pipeline.core'; Version = '4.5.131' + Previous = $null; Reason = 'changed' + } + [pscustomobject]@{ + Name = 'pipeline.engines'; Version = '4.5.131' + Previous = $null; Reason = 'depends on pipeline.core' + } + } + } + + It 'pushes every tag in one atomic push' { + Invoke-Publish + Should -Invoke git -Times 1 -Exactly -ParameterFilter { + $args -contains 'push' + } + Should -Invoke git -Times 1 -Exactly -ParameterFilter { + $args -contains 'push' -and $args -contains '--atomic' -and + $args -contains 'pipeline.core/4.5.131' -and + $args -contains 'pipeline.engines/4.5.131' + } + } + + It 'makes one release for each tag' { + Invoke-Publish + Should -Invoke gh -Times 2 -Exactly + $global:SkipUpdateTag | Should -BeTrue + } + + It 'makes no release when the push fails' { + Mock git { if ($args -contains 'push') { throw 'push refused' } } + { Invoke-Publish } | Should -Throw '*push refused*' + Should -Invoke gh -Times 0 -Exactly + } + + It 'pushes nothing and makes no release in a dry run' { + Invoke-Publish -DryRun $true + Should -Invoke git -Times 2 -Exactly -ParameterFilter { + $args -contains 'tag' + } + Should -Invoke git -Times 0 -Exactly -ParameterFilter { + $args -contains 'push' + } + Should -Invoke gh -Times 0 -Exactly + $global:SkipUpdateTag | Should -BeTrue + } + + It 'pushes nothing when the plan is empty' { + Mock Get-ReleasePlan { } + Invoke-Publish + Should -Invoke git -Times 0 -Exactly -ParameterFilter { + $args -contains 'push' + } + Should -Invoke gh -Times 0 -Exactly + } +} diff --git a/ci/publish-package.ps1 b/ci/publish-package.ps1 index f2ee093f3..59401d1ed 100644 --- a/ci/publish-package.ps1 +++ b/ci/publish-package.ps1 @@ -1,12 +1,60 @@ param( [Parameter(Mandatory)][string]$RepoName, + [Parameter(Mandatory)][string]$OrgName, [Parameter(Mandatory)][string]$Version, [Parameter(Mandatory)][Hashtable]$Keys, [bool]$DryRun = $false ) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true -# Uploads the one publishing bundle build-package.ps1 made, which holds the -# packages of every tree that publishes. +. "$PSScriptRoot/release.ps1" + +# The plan is read before anything is tagged, because the tags made below +# change what the plan would say. +$plan = @(Get-ReleasePlan $RepoName) + +# Uploads the one publishing bundle build-package.ps1 made, which holds every +# package this publish releases. ./java/publish-package-maven.ps1 -RepoName $RepoName -MavenSettings $Keys['MavenSettings'] -Version $Version -DryRun $DryRun -exit $LASTEXITCODE +# One tag and one GitHub release per package, '/', in place +# of common-ci's single tag, which would name no package. The tags go up in one +# atomic push, so a publish that stops leaves every package tagged or none. +# A release tagged in part would leave the untagged dependants out of the plan +# of a run made again, because the plan skips a package tagged here and finds +# dependants only from packages that changed. +$tags = @($plan | ForEach-Object { "$($_.Name)/$($_.Version)" }) +foreach ($tag in $tags) { + git -C $RepoName tag $tag +} +if ($tags.Count -eq 0 -or $DryRun -eq $true) { + # A push with no tags named would push the branch instead. + Write-Output "Dry run or empty plan - tagged '$($tags -join ', ')' locally, not pushed, no release made" + $global:SkipUpdateTag = $true + return +} +Write-Output "Pushing $($tags -join ', ')" +git -C $RepoName push --atomic origin @tags + +# The notes list the commits that touched the package's files since its last +# release. +foreach ($release in $plan) { + $tag = "$($release.Name)/$($release.Version)" + $notes = "$($release.Name) $($release.Version): $($release.Reason)." + if ($null -ne $release.Previous) { + $package = Get-PublishedPackages $RepoName | + Where-Object { $_.Name -eq $release.Name } + $range = "$($release.Name)/$($release.Previous)..HEAD" + $commits = git -C $RepoName log --format='- %s (%h)' $range ` + -- (Get-PackagePaths $package) + if ($commits) { + $notes += "`n`n" + ($commits -join "`n") + } + } + gh -R "$OrgName/$RepoName" release create $tag --verify-tag ` + --title "$($release.Name) $($release.Version)" --notes $notes +} + +# Tells common-ci's publish not to add its own single tag and release. +$global:SkipUpdateTag = $true diff --git a/ci/release-plan-filter.Tests.ps1 b/ci/release-plan-filter.Tests.ps1 new file mode 100644 index 000000000..62a854a43 --- /dev/null +++ b/ci/release-plan-filter.Tests.ps1 @@ -0,0 +1,161 @@ +# Pester 5 tests of release-plan-filter.ps1. Each test makes a repository +# whose pom.xml is tagged as released, changes it in a second commit, and asks +# the filter whether the change counts. + +BeforeAll { + $Filter = Join-Path $PSScriptRoot 'release-plan-filter.ps1' + + $Released = @' + + + 4.0.0 + com.51degrees + pipeline.core + 4.5.130 + 51Degrees :: Pipeline :: Core + Pipeline core + https://example.com + + EUPL-1.2 + + + 51Degrees + + + https://github.com/51Degrees/pipeline-java + + + https://github.com/51Degrees/pipeline-java/issues + + + + central + https://repo1.maven.org/maven2 + + + + + org.json + json + 20240303 + + + + + + maven-jar-plugin + 3.3.0 + + + + +'@ + + # A repository holding $Released at the tag 'pkg/4.5.130', then the given + # files in a second commit. Returns the repository's folder. + function New-ChangedRepo([hashtable]$Files, [string[]]$Remove = @()) { + $repo = Join-Path $TestDrive ([Guid]::NewGuid().ToString('N')) + $null = New-Item -ItemType Directory -Path (Join-Path $repo 'pkg') + Set-Content -LiteralPath (Join-Path $repo 'pkg/pom.xml') -Value $Released -NoNewline + Set-Content -LiteralPath (Join-Path $repo 'pkg/Core.java') -Value 'class Core {}' -NoNewline + git -C $repo init -q + git -C $repo config user.email 'test@example.com' + git -C $repo config user.name 'Test' + git -C $repo config core.autocrlf false + git -C $repo add -A + git -C $repo commit -q -m 'released' + git -C $repo tag 'pkg/4.5.130' + foreach ($path in $Files.Keys) { + $full = Join-Path $repo $path + $null = New-Item -ItemType Directory -Force -Path (Split-Path $full) + Set-Content -LiteralPath $full -Value $Files[$path] -NoNewline + } + foreach ($path in $Remove) { + git -C $repo rm -q $path + } + git -C $repo add -A + git -C $repo commit -q --allow-empty -m 'changed' + return $repo + } + + # The released pom with one piece of text replaced, which must be there. + function Edit-Released([string]$Old, [string]$New) { + if ($Released.Contains($Old) -eq $false) { + throw "The released pom has no '$Old'." + } + return $Released.Replace($Old, $New) + } + + function Invoke-Filter([string]$Repo, [string[]]$Files) { + return @(& $Filter -RepoPath $Repo -Package 'pkg' -From 'pkg/4.5.130' -Files $Files) + } +} + +Describe 'release-plan-filter.ps1' { + It 'passes a file that is not a pom.xml through' { + $repo = New-ChangedRepo @{ 'pkg/Core.java' = 'class Core { int a; }' } + Invoke-Filter $repo @('pkg/Core.java') | Should -Be @('pkg/Core.java') + } + + It 'leaves out a pom.xml whose changed' -ForEach @( + @{ Metadata = 'name'; Old = 'Pipeline :: Core'; New = 'Pipeline :: Core API' } + @{ Metadata = 'description'; Old = 'Pipeline core'; New = 'The Pipeline core' } + @{ Metadata = 'url'; Old = 'https://example.com'; New = 'https://example.com/documentation' } + @{ Metadata = 'licenses'; Old = 'EUPL-1.2'; New = 'EUPL 1.2' } + @{ Metadata = 'developers'; Old = '51Degrees'; New = '51Degrees Mobile Experts' } + @{ Metadata = 'scm'; Old = 'pipeline-java'; New = '51degrees-java' } + @{ Metadata = 'issueManagement'; Old = 'pipeline-java/issues'; New = '51degrees-java/issues' } + ) { + $repo = New-ChangedRepo @{ 'pkg/pom.xml' = (Edit-Released $Old $New) } + Invoke-Filter $repo @('pkg/pom.xml') | Should -BeNullOrEmpty + } + + It 'leaves out a pom.xml whose only change is a comment or its layout' { + $pom = Edit-Released '4.0.0' ` + "`n 4.0.0`n" + $repo = New-ChangedRepo @{ 'pkg/pom.xml' = $pom } + Invoke-Filter $repo @('pkg/pom.xml') | Should -BeNullOrEmpty + } + + It 'keeps a pom.xml whose changed' -ForEach @( + @{ Change = 'version'; Old = '4.5.130'; New = '4.5.131' } + @{ Change = 'dependency version'; Old = '20240303'; New = '20250107' } + @{ Change = 'dependencies'; Old = ''; New = 'org.slf4jslf4j-api' } + @{ Change = 'plugin'; Old = '3.3.0'; New = '3.4.1' } + @{ Change = 'build'; Old = ''; New = 'core' } + @{ Change = 'repository url'; Old = 'https://repo1.maven.org/maven2'; New = 'https://repo.maven.apache.org/maven2' } + ) { + $repo = New-ChangedRepo @{ 'pkg/pom.xml' = (Edit-Released $Old $New) } + Invoke-Filter $repo @('pkg/pom.xml') | Should -Be @('pkg/pom.xml') + } + + It 'keeps a pom.xml whose metadata and dependencies both changed' { + $pom = (Edit-Released 'Pipeline core' 'The Pipeline core'). + Replace('20240303', '20250107') + $repo = New-ChangedRepo @{ 'pkg/pom.xml' = $pom } + Invoke-Filter $repo @('pkg/pom.xml') | Should -Be @('pkg/pom.xml') + } + + It 'keeps a pom.xml that is new since the release' { + $repo = New-ChangedRepo @{ 'pkg/sub/pom.xml' = $Released } + Invoke-Filter $repo @('pkg/sub/pom.xml') | Should -Be @('pkg/sub/pom.xml') + } + + It 'keeps a pom.xml that was removed since the release' { + $repo = New-ChangedRepo @{} -Remove @('pkg/pom.xml') + Invoke-Filter $repo @('pkg/pom.xml') | Should -Be @('pkg/pom.xml') + } + + It 'judges each file on its own' { + $repo = New-ChangedRepo @{ + 'pkg/pom.xml' = (Edit-Released 'Pipeline core' 'The Pipeline core') + 'pkg/Core.java' = 'class Core { int a; }' + } + Invoke-Filter $repo @('pkg/pom.xml', 'pkg/Core.java') | Should -Be @('pkg/Core.java') + } + + It 'gives nothing back for no files' { + $repo = New-ChangedRepo @{} + Invoke-Filter $repo @() | Should -BeNullOrEmpty + } +} diff --git a/ci/release-plan-filter.ps1 b/ci/release-plan-filter.ps1 new file mode 100644 index 000000000..7561498f1 --- /dev/null +++ b/ci/release-plan-filter.ps1 @@ -0,0 +1,70 @@ +# Called by common-ci's steps/release-plan.ps1 with the files under a +# package's folders that changed since its last tag, once its Ignore globs are +# applied. Writes out the files whose change releases the package. +# +# A pom.xml counts only when something other than the project's descriptive +# metadata changed, so a new link, description or developer does not release +# the package, while a version, dependency, plugin or build change does. +# Comments and layout do not count either. Every other file counts. +param( + [Parameter(Mandatory)][string]$RepoPath, + # The package the files belong to. Not needed here, because a pom.xml is + # judged the same way whichever package it belongs to. + $Package, + # The revision the package was last released from, normally its tag. + [Parameter(Mandatory)][string]$From, + [string[]]$Files = @() +) +$ErrorActionPreference = 'Stop' +# 'git show' fails for a file that is not in a revision, which is an answer +# here rather than an error. +$PSNativeCommandUseErrorActionPreference = $false + +# The children of that describe the project rather than build it. +$MetadataElements = @( + 'name', 'description', 'url', 'licenses', 'developers', 'scm', + 'issueManagement') + +# The pom.xml at a revision with its comments, its layout and its metadata +# taken out, or $null when the revision has no such file or it is not XML. +function Get-PomBuildText([string]$Revision, [string]$File) { + # git's message for a missing file is not an error of this script. + $ErrorActionPreference = 'Continue' + $text = & git -C $RepoPath show "${Revision}:$File" 2>$null + $found = $LASTEXITCODE -eq 0 + $global:LASTEXITCODE = 0 + if ($found -eq $false) { + return $null + } + $xml = [Xml.XmlDocument]::new() + $xml.PreserveWhitespace = $false + try { + $xml.LoadXml(($text -join "`n")) + } catch { + return $null + } + foreach ($comment in @($xml.SelectNodes('//comment()'))) { + $null = $comment.ParentNode.RemoveChild($comment) + } + $project = $xml.DocumentElement + foreach ($child in @($project.ChildNodes)) { + if ($child.NodeType -eq [Xml.XmlNodeType]::Element -and + $MetadataElements -contains $child.LocalName) { + $null = $project.RemoveChild($child) + } + } + return $project.OuterXml +} + +foreach ($file in $Files) { + if (($file -replace '\\', '/') -notmatch '(^|/)pom\.xml$') { + $file + continue + } + $before = Get-PomBuildText $From $file + $after = Get-PomBuildText 'HEAD' $file + # A pom.xml that is new, removed or unreadable on either side counts. + if ($null -eq $before -or $null -eq $after -or $before -ne $after) { + $file + } +} diff --git a/ci/release.ps1 b/ci/release.ps1 new file mode 100644 index 000000000..13bfa8501 --- /dev/null +++ b/ci/release.ps1 @@ -0,0 +1,125 @@ +# Dot-sourced by the scripts that publish. Each package in packages.json has +# its own version, tagged '/'. A publish releases a package +# when a file under its folders changed since its last tag, and every package +# that depends on it, with the patch number moved. common-ci's +# steps/release-plan.ps1 works out the plan, and the functions here build it. +# +# Kept apart from packages.ps1, which every script loads, so that building and +# testing a pull request does not need the release plan. +. "$PSScriptRoot/packages.ps1" + +# common-ci runs these scripts from its own root, with this repository checked +# out inside it. A checkout of common-ci beside this repository is found too, +# or one elsewhere can be named with COMMON_CI. +$releasePlan = @( + $env:COMMON_CI, + (Join-Path $PSScriptRoot '..' '..'), + (Join-Path $PSScriptRoot '..' '..' 'common-ci')) | + Where-Object { $_ } | + ForEach-Object { Join-Path $_ 'steps' 'release-plan.ps1' } | + Where-Object { Test-Path $_ } | + Select-Object -First 1 +if ($null -eq $releasePlan) { + throw "common-ci's steps/release-plan.ps1 was not found. Set COMMON_CI " + + "to a checkout of common-ci." +} +. $releasePlan +Remove-Variable releasePlan + +# The examples' properties that name the version of a package they use, which +# the publish sets to the version it built. +$ExampleVersionProperties = @{ + 'device-detection.version' = 'device-detection' + 'ip-intelligence.version' = 'ip-intelligence' + 'pipeline.jsonbuilder.version' = 'pipeline.jsonbuilder' +} + +# The packages a publish builds. When nothing is to be released, every package +# is built at its released version, so that the tests have packages to install +# and the nightly publish passes without publishing anything. +function Get-BuildPlan([string]$RepoPath) { + $plan = @(Get-ReleasePlan $RepoPath) + if ($plan.Count -gt 0) { + return $plan + } + $published = Get-PublishedPackages $RepoPath + foreach ($name in (Sort-ByDependency $published @($published.Name))) { + $package = $published | Where-Object { $_.Name -eq $name } + $last = @(Get-ReleasedVersions $RepoPath $name)[-1] + [pscustomobject]@{ + Name = $name + Path = $package.Path + Version = "$last" + Previous = $last + Reason = 'already released, built for the tests' + } + } +} + +# The version of every published package in a build, which is its version in +# the plan, or its last release when the plan leaves it out. +function Get-BuildVersions([string]$RepoPath, $Plan = $null) { + if ($null -eq $Plan) { + $Plan = @(Get-BuildPlan $RepoPath) + } + $versions = @{} + foreach ($package in Get-PublishedPackages $RepoPath) { + $release = $Plan | Where-Object { $_.Name -eq $package.Name } + if ($null -ne $release) { + $versions[$package.Name] = $release.Version + continue + } + $released = @(Get-ReleasedVersions $RepoPath $package.Name) + if ($released.Count -eq 0) { + throw "$($package.Name) has never been released and is not in the plan." + } + $versions[$package.Name] = "$($released[-1])" + } + return $versions +} + +# Writes into every published pom.xml its own version, its parent's and those +# of the packages it depends on, so each module is built at its own version and +# its published pom requires the versions it was built with. The poms in the +# repository keep one development version for every module, which this +# replaces only in the checkout being built. +function Set-PomVersions([string]$RepoPath, [hashtable]$Versions) { + $gaps = @(Get-PackagesJsonGaps $RepoPath) + if ($gaps.Count -gt 0) { + throw "ci/packages.json does not match the poms:`n$($gaps -join "`n")" + } + $packages = Get-PublishedPackages $RepoPath + $names = @($packages.Name) + foreach ($package in $packages) { + $pom = Read-Pom (Join-Path $RepoPath (Get-PomPath $package)) + $project = $pom.Xml.DocumentElement + $version = $project.SelectSingleNode('m:version', $pom.Ns) + if ($null -eq $version) { + # The module took its version from its parent. It gets an element + # of its own after its artifactId, indented as the artifactId is. + $artifactId = $project.SelectSingleNode('m:artifactId', $pom.Ns) + $version = $pom.Xml.CreateElement('version', $PomNamespace) + $null = $project.InsertAfter($version, $artifactId) + $indent = $artifactId.PreviousSibling + if ($null -ne $indent -and + $indent.NodeType -eq [Xml.XmlNodeType]::Whitespace) { + $null = $project.InsertAfter($indent.Clone(), $artifactId) + } + } + $version.InnerText = $Versions[$package.Name] + foreach ($reference in Get-PomReferences $pom $names) { + if ($null -ne $reference.Version) { + $reference.Version.InnerText = $Versions[$reference.Name] + } + } + Save-Pom $pom + } +} + +# The arguments that point the examples at the versions a publish built. +function Get-ExampleVersionArguments([string]$RepoPath) { + $versions = Get-BuildVersions $RepoPath + foreach ($property in $ExampleVersionProperties.Keys | Sort-Object) { + "-D$property=$($versions[$ExampleVersionProperties[$property]])" + } +} diff --git a/ci/run-fresh-clone-example.Tests.ps1 b/ci/run-fresh-clone-example.Tests.ps1 new file mode 100644 index 000000000..28deef366 --- /dev/null +++ b/ci/run-fresh-clone-example.Tests.ps1 @@ -0,0 +1,172 @@ +# Tests for ci/run-fresh-clone-example.ps1. +# +# Each case runs the script in a new pwsh process, with -File as a person +# would type it at a shell, or with -Command as the workflow's pwsh step does. +# The process runs in a folder that holds an empty jar where the script looks +# for one, and a stand-in for java comes first on the PATH. The stand-in +# prints results, and also the line an example logs when it downloads its data +# file when it is given the example name Downloads. + +BeforeDiscovery { + # Each call the workflow makes, with the job it is in, so that a change to + # the workflow's call is tested as well. + $workflow = Join-Path $PSScriptRoot '../.github/workflows/examples-fresh-clone.yml' + $lines = @(Get-Content -LiteralPath $workflow) + $script:WorkflowCalls = @() + for ($i = 0; $i -lt $lines.Count - 1; $i++) { + if ($lines[$i].Trim() -ne 'run: >-' -or + $lines[$i + 1] -notmatch 'run-fresh-clone-example\.ps1') { + continue + } + $parts = @() + for ($j = $i + 1; $j -lt $lines.Count -and $lines[$j].Trim() -ne ''; $j++) { + $parts += $lines[$j].Trim() + } + $job = $null + for ($k = $i; $k -ge 0 -and $null -eq $job; $k--) { + if ($lines[$k] -match '^ ([A-Za-z0-9_-]+):\s*$') { + $job = $Matches[1] + } + } + $script:WorkflowCalls += @{ Job = $job; Command = $parts -join ' ' } + } +} + +BeforeAll { + $script:Script = Join-Path $PSScriptRoot 'run-fresh-clone-example.ps1' + $script:Pwsh = (Get-Process -Id $PID).Path + + $script:Root = Join-Path $TestDrive 'repository' + $target = Join-Path $script:Root 'examples/device-detection/console/target' + New-Item -ItemType Directory $target -Force | Out-Null + New-Item -ItemType File (Join-Path $target 'example-jar-with-dependencies.jar') | Out-Null + + $script:Bin = Join-Path $TestDrive 'bin' + New-Item -ItemType Directory $script:Bin | Out-Null + $missing = "'device-detection/51Degrees-LiteV4.1.hash' is missing, so it is being downloaded by running the script" + if ($IsWindows) { + # The arguments are -cp, the jar and the example. + [IO.File]::WriteAllText((Join-Path $script:Bin 'java.cmd'), + "@echo off`r`nif ""%~3""==""Downloads"" echo $missing`r`necho Results:`r`nexit /b 0`r`n") + } else { + $java = Join-Path $script:Bin 'java' + [IO.File]::WriteAllText($java, + "#!/bin/sh`nif [ ""`$3"" = Downloads ]; then echo ""$missing""; fi`necho Results:`nexit 0`n") + chmod +x $java + } + + # Runs pwsh with the arguments in the repository folder, with the stand-in + # for java first on the PATH, and returns the exit code and the output. + function Invoke-Pwsh([string[]]$Arguments) { + $path = $env:PATH + Push-Location $script:Root + try { + $env:PATH = $script:Bin + [IO.Path]::PathSeparator + $path + $output = & $script:Pwsh -NoProfile -NonInteractive @Arguments 2>&1 | + ForEach-Object { "$_" } + [pscustomobject]@{ Code = $LASTEXITCODE; Text = $output -join "`n" } + } finally { + $env:PATH = $path + Pop-Location + } + } + + function Invoke-File([string]$Example, [string[]]$Flag) { + Invoke-Pwsh (@('-File', $script:Script, '-Product', 'device-detection', + '-Example', $Example) + $Flag) + } +} + +Describe 'run-fresh-clone-example.ps1 called with pwsh -File' { + + It 'passes when the example downloads, given ' -ForEach @( + @{ Flag = @('-ExpectDownload') } + @{ Flag = @('-ExpectDownload:$true') } + ) { + $run = Invoke-File 'Downloads' $Flag + $run.Text | Should -Match 'Results:' + $run.Text | Should -Not -Match '::error::' + $run.Code | Should -Be 0 + } + + It 'fails when the example does not download, given ' -ForEach @( + @{ Flag = @('-ExpectDownload') } + @{ Flag = @('-ExpectDownload:$true') } + ) { + $run = Invoke-File 'FromCache' $Flag + $run.Text | Should -Match '::error::The example did not download its free data file\.' + $run.Code | Should -Be 1 + } + + It 'passes when the example does not download, given ' -ForEach @( + @{ Name = 'no flag'; Flag = @() } + @{ Name = '-ExpectDownload:$false'; Flag = @('-ExpectDownload:$false') } + ) { + $run = Invoke-File 'FromCache' $Flag + $run.Text | Should -Match 'Results:' + $run.Text | Should -Not -Match '::error::' + $run.Code | Should -Be 0 + } + + It 'fails when the example downloads, given ' -ForEach @( + @{ Name = 'no flag'; Flag = @() } + @{ Name = '-ExpectDownload:$false'; Flag = @('-ExpectDownload:$false') } + ) { + $run = Invoke-File 'Downloads' $Flag + $run.Text | Should -Match '::error::The example downloaded its data file' + $run.Code | Should -Be 1 + } +} + +Describe 'run-fresh-clone-example.ps1 given text the output must hold' { + + It 'passes when the output holds the text' { + $run = Invoke-File 'FromCache' @('-Expect', 'Results:') + $run.Text | Should -Not -Match '::error::' + $run.Code | Should -Be 0 + } + + It 'fails when the output does not hold the text' { + $run = Invoke-File 'FromCache' @('-Expect', '(weighting ') + $run.Text | Should -Match "::error::The example's output does not hold '\(weighting '\." + $run.Code | Should -Be 1 + } +} + +Describe 'run-fresh-clone-example.ps1 called as the workflow calls it' { + + It 'finds both of the workflow''s calls' -ForEach @( + @{ Found = $script:WorkflowCalls.Count } + ) { + $Found | Should -Be 2 + } + + # The download job expects the example to download its file, and the + # other jobs expect it to be there already. + # The command is wrapped as GitHub's pwsh shell wraps a step, which stops + # on an error and exits with the last exit code. + It 'passes the job''s call only when the example does what that job expects' -ForEach $script:WorkflowCalls { + $quoted = $script:Script.Replace("'", "''") + $call = $Command.Replace('./ci/run-fresh-clone-example.ps1', "& '$quoted'") + $call = $call.Replace('${{ matrix.product }}', 'device-detection') + # The stand-in prints no weighting, so the expected text is the one + # the device-detection entries give, which is none. + $call = $call.Replace('${{ matrix.expect }}', '') + $call = "`$ErrorActionPreference = 'stop'`n$call`n" + + "if ((Test-Path -LiteralPath variable:\LASTEXITCODE)) { exit `$LASTEXITCODE }" + $expected, $other = if ($Job -eq 'download') { + 'Downloads', 'FromCache' + } else { + 'FromCache', 'Downloads' + } + + $run = Invoke-Pwsh @('-Command', $call.Replace('${{ matrix.example }}', $expected)) + $run.Text | Should -Match 'Results:' + $run.Text | Should -Not -Match '::error::' + $run.Code | Should -Be 0 + + $run = Invoke-Pwsh @('-Command', $call.Replace('${{ matrix.example }}', $other)) + $run.Text | Should -Match '::error::' + $run.Code | Should -Be 1 + } +} diff --git a/ci/run-fresh-clone-example.ps1 b/ci/run-fresh-clone-example.ps1 new file mode 100644 index 000000000..c3d9474ee --- /dev/null +++ b/ci/run-fresh-clone-example.ps1 @@ -0,0 +1,53 @@ +param ( + [Parameter(Mandatory)][string]$Product, + [Parameter(Mandatory)][string]$Example, + # Given where the clone holds only the data file's link, so the example + # has to download the file. Left out, or given as -ExpectDownload:$false, + # where the file is already in place. A switch, so that it works the same + # with pwsh -File, which passes every other value as a string. + [switch]$ExpectDownload, + # Text the output must hold, such as the weighting the IP Intelligence + # example prints with each value. Empty when there is none. + [string]$Expect = '' +) + +# Runs a product's console example jar from the repository root with no +# settings, as AGENTS.md's "Run the examples" does, and fails unless it ends +# with exit code 0, prints results and any expected text, and downloads its +# data file exactly when expected. + +$jar = Get-ChildItem "examples/$Product/console/target" ` + -Filter '*-jar-with-dependencies.jar' | Select-Object -First 1 +if ($null -eq $jar) { + Write-Output '::error::The build made no jar with dependencies.' + exit 1 +} +$output = & java -cp $jar.FullName $Example 2>&1 | ForEach-Object { "$_" } +$code = $LASTEXITCODE +$output +$text = $output -join "`n" +$failed = $false +if ($code -ne 0) { + Write-Output "::error::The example exited with $code." + $failed = $true +} +$downloaded = $text -match 'is missing, so it is being downloaded' +if ($ExpectDownload -and -not $downloaded) { + Write-Output '::error::The example did not download its free data file.' + $failed = $true +} +if (-not $ExpectDownload -and $downloaded) { + Write-Output '::error::The example downloaded its data file, which should have come from the cache.' + $failed = $true +} +if ($text -notmatch 'Results:') { + Write-Output '::error::The example printed no results.' + $failed = $true +} +if ($Expect -and -not $text.Contains($Expect)) { + Write-Output "::error::The example's output does not hold '$Expect'." + $failed = $true +} +if ($failed) { + exit 1 +} diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index ff9175676..2df9e505f 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -1,13 +1,13 @@ param( [Parameter(Mandatory)][string]$RepoName, [string]$Name, - # Set by the publish workflow to the version it built and installed. Not - # set when a pull request is tested, which builds the reactor itself. + # Set by the publish workflow, which built and installed the packages, to + # its label for the run. Not set when a pull request is tested, which + # builds the reactor itself. [string]$Version, - [string]$TestResourceKey, - [string]$DeviceDetection, - [string]$IpIntelligence, - [string]$IpIntelligenceUrl + # False, from select-options.ps1, on every entry of a pull request but + # one, so the browser contract runs once. + [bool]$SeleniumContract = $true ) $ErrorActionPreference = "Stop" # Each part's exit code is checked below, so one failing part does not stop @@ -20,23 +20,24 @@ $results = "$RepoName/test-results/integration" $failed = @() # The examples take the libraries from Maven Central unless told otherwise. -# Pointing them at the version the publish workflow installed tests the -# packages as a project of your own would use them. For a pull request the -# dev profile points them at the reactor build-project.ps1 installed. -[string[]]$versionArgs = if ($Version) { - "-Ddevice-detection.version=$Version", "-Dip-intelligence.version=$Version" +# Pointing them at the versions the publish workflow built and installed, each +# package's own, tests the packages as a project of your own would use them. +# For a pull request the dev profile points them at the reactor +# build-project.ps1 installed. +if ($Version) { + . "$PSScriptRoot/release.ps1" + [string[]]$versionArgs = Get-ExampleVersionArguments $RepoName } else { - , '-Pdev' + [string[]]$versionArgs = '-Pdev' } # The Pipeline API developer examples, as the pipeline repository ran them. Write-Host "::group::Pipeline developer examples" Push-Location "$RepoName/pipeline" try { - # The cloud engine example calls the live cloud service, so it is given - # the test resource key, as the unit test run gives it. - mvn test @MavenOptions -DfailIfNoTests=false "-Dsurefire.failIfNoSpecifiedTests=false" -Dtest=ExampleTests ` - "-DTestResourceKey=$TestResourceKey" + # The cloud engine example calls the live cloud service, with the key in + # _51DEGREES_RESOURCE_KEY. + mvn test @MavenOptions -DfailIfNoTests=false "-Dsurefire.failIfNoSpecifiedTests=false" -Dtest=ExampleTests if ($LASTEXITCODE -ne 0) { $failed += 'pipeline developer examples' } } finally { Pop-Location @@ -47,29 +48,19 @@ Write-Host "::endgroup::" # The product examples. A pull request has already run them in the unit tests, # against the reactor, so they are run here only against a published version. if ($Version) { - $examples = @( - @{ Path = 'examples/device-detection'; LicenseKey = $DeviceDetection; Run = $true } - # The IP Intelligence examples need the Enterprise data file, which is - # only fetched when its URL is given. - @{ Path = 'examples/ip-intelligence'; LicenseKey = $IpIntelligence; Run = [bool]$IpIntelligenceUrl } - ) + # The tests read their keys from the _51DEGREES_ environment variables, + # and the Enterprise data files were fetched with _51DEGREES_LICENSE_KEY. + $examples = @('examples/device-detection', 'examples/ip-intelligence') foreach ($example in $examples) { - if (-not $example.Run) { - Write-Host "::warning title=Examples skipped::Skipping '$($example.Path)', as IpIntelligenceUrl was not provided." - continue - } - Write-Host "::group::$($example.Path) against $Version" - Push-Location "$RepoName/$($example.Path)" + Write-Host "::group::$example against $($versionArgs -join ' ')" + Push-Location "$RepoName/$example" try { - mvn clean test @MavenOptions @versionArgs ` - "-DTestResourceKey=$TestResourceKey" ` - "-DSuperResourceKey=$TestResourceKey" ` - "-DLicenseKey=$($example.LicenseKey)" - if ($LASTEXITCODE -ne 0) { $failed += $example.Path } + mvn clean test @MavenOptions @versionArgs + if ($LASTEXITCODE -ne 0) { $failed += $example } } finally { Pop-Location } - & "$PSScriptRoot/copy-test-results.ps1" -Path "$RepoName/$($example.Path)" -Destination $results + & "$PSScriptRoot/copy-test-results.ps1" -Path "$RepoName/$example" -Destination $results Write-Host "::endgroup::" } } else { @@ -111,7 +102,8 @@ function Invoke-ContractTests { $env:EXAMPLE_URL = "http://localhost:$Port" $env:EXAMPLE_LANG = 'java' - dotnet test selenium-api-tests -c Release --filter TestCategory=Contract + # Normal verbosity prints each test's time, passed or failed. + dotnet test selenium-api-tests -c Release --filter TestCategory=Contract --logger "console;verbosity=normal" # PowerShell does not stop for a native command that returns non-zero, # so without this the browser tests could fail every case and the job # would still be reported as a success. @@ -131,8 +123,10 @@ function Invoke-ContractTests { } Write-Host "::group::Selenium contract tests" -if (-not $TestResourceKey) { - Write-Host "::warning title=Selenium skipped::No resource key, so the Selenium contract tests were not run against the web examples." +if (-not $SeleniumContract) { + Write-Host "Another entry runs the Selenium contract tests for this pull request." +} elseif (-not $env:_51DEGREES_RESOURCE_KEY) { + Write-Host "::warning title=Selenium skipped::_51DEGREES_RESOURCE_KEY is not set, so the Selenium contract tests were not run against the web examples." } elseif ($IsLinux -and [System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture -eq [System.Runtime.InteropServices.Architecture]::Arm64) { Write-Host "::warning title=Selenium skipped::Selenium contract skipped on linux-arm64, which has no selenium-manager build. It runs on x64 and macOS arm64." } else { @@ -144,17 +138,25 @@ if (-not $TestResourceKey) { git clone --depth 1 --branch $seleniumRef https://github.com/51Degrees/selenium-api-tests.git } $env:CLOUD_ROOT_URL = "https://cloud.51degrees.com/" - $env:PAID_RESOURCE_KEY = $TestResourceKey + # The shared suite reads its key under this name of its own. + $env:PAID_RESOURCE_KEY = $env:_51DEGREES_RESOURCE_KEY + + # Chrome's first start on a fresh runner is many times slower than the + # starts after it, so it is made here rather than inside the first test. + # The tests still run when it cannot be made. + try { + & "$PSScriptRoot/start-chrome-once.ps1" + } catch { + Write-Host "::warning title=Chrome not started::$_" + } $contracts = @( @{ Label = 'cloud' Module = 'web/getting-started.cloud' + # The example reads _51DEGREES_RESOURCE_KEY, and the production + # cloud unless _51DEGREES_CLOUD_ENDPOINT names another. Port = 8099 - ExampleEnv = @{ - TestCloudEndpoint = "https://cloud.51degrees.com/api/v4" - TestResourceKey = $TestResourceKey - } } # The on-premise example, against the TAC data file that # fetch-assets.ps1 links into the data repository. The Lite data file diff --git a/ci/run-performance-tests.ps1 b/ci/run-performance-tests.ps1 index 9b24f7024..f2fe18f9e 100644 --- a/ci/run-performance-tests.ps1 +++ b/ci/run-performance-tests.ps1 @@ -1,10 +1,10 @@ param( [Parameter(Mandatory)][string]$RepoName, [Parameter(Mandatory)][string]$Name, - # Set by the publish workflow to the version it built and installed. Not - # set when a pull request is tested, which builds the reactor itself. - [string]$Version, - [string]$IpIntelligenceUrl + # Set by the publish workflow, which built and installed the packages, to + # its label for the run. Not set when a pull request is tested, which + # builds the reactor itself. + [string]$Version ) $ErrorActionPreference = "Stop" # Each product's exit code is checked below, so one failing product does not @@ -21,10 +21,12 @@ $summary = @{ HigherIsBetter = [ordered]@{}; LowerIsBetter = [ordered]@{} } $results = "$RepoName/test-results/performance" $failed = @() -[string[]]$versionArgs = if ($Version) { - "-Ddevice-detection.version=$Version", "-Dip-intelligence.version=$Version" +# A publish measures the versions it built, each package's own. +if ($Version) { + . "$PSScriptRoot/release.ps1" + [string[]]$versionArgs = Get-ExampleVersionArguments $RepoName } else { - , '-Pdev' + [string[]]$versionArgs = '-Pdev' } # Pipeline API. The test writes its figures as JSON itself. @@ -103,7 +105,6 @@ $benchmarks = @( Output = 'console/target/surefire-reports/fiftyone.devicedetection.examples.console.PerformanceBenchmarkTest-output.txt' ProfileHeader = "MaxPerformance AllProperties: (true|false), performanceGraph: (true|false), predictiveGraph (true|false)" ProfileName = 'MaxPerformance-false-true-false' - Run = $true } @{ Product = 'IpIntelligence' @@ -111,17 +112,12 @@ $benchmarks = @( Output = 'console/target/surefire-reports/fiftyone.ipintelligence.examples.console.PerformanceBenchmarkTest-output.txt' ProfileHeader = "MaxPerformance AllProperties: (true|false)" ProfileName = 'MaxPerformance-false' - # Measured with the Enterprise data file, which is only fetched when - # its URL is given. - Run = [bool]$IpIntelligenceUrl + # Measured with the Enterprise data file, which common-ci fetches + # with _51DEGREES_LICENSE_KEY. } ) foreach ($benchmark in $benchmarks) { - if (-not $benchmark.Run) { - Write-Host "::warning title=Performance skipped::Skipping the $($benchmark.Product) benchmark, as IpIntelligenceUrl was not provided." - continue - } Write-Host "::group::$($benchmark.Product)" Push-Location "$RepoName/$($benchmark.Path)" try { diff --git a/ci/run-unit-tests.ps1 b/ci/run-unit-tests.ps1 index a54755337..08fa78c43 100644 --- a/ci/run-unit-tests.ps1 +++ b/ci/run-unit-tests.ps1 @@ -1,8 +1,10 @@ param( [Parameter(Mandatory)][string]$RepoName, [string]$Name, - [string]$TestResourceKey, - [string]$DeviceDetection + # How deeply a pull request is tested, as select-options.ps1 chose. + # standard leaves out the tests listed in standard-excludes.txt. Empty, for + # the branch itself or a release, runs every test. + [string]$Depth ) $ErrorActionPreference = "Stop" @@ -10,15 +12,19 @@ $ErrorActionPreference = "Stop" # examples included. common-ci runs Surefire from the root, copies the reports # of every module at any depth into test-results/unit, leaving out the example # tests the integration tests report, and logs why a forked JVM died when one -# does. The dev profile points the examples at the reactor. SuperResourceKey is -# read by the Location and Device Detection examples tests, and LicenseKey by -# the Device Detection data file update example. The IP Intelligence update -# example's test is ignored, so it does not need a key of its own. -./java/run-unit-tests.ps1 -RepoName $RepoName -Name $Name -ExtraArgs @( +# does. The dev profile points the examples at the reactor. The tests read +# their keys from the environment, where common-ci has put them under their +# _51DEGREES_ names: _51DEGREES_RESOURCE_KEY, _51DEGREES_RESOURCE_KEY_SUPER +# for the Location and the TAC and native model tests, and +# _51DEGREES_LICENSE_KEY for the Device Detection data file update example. +# A pull request at the standard depth leaves out the tests in +# standard-excludes.txt. +$extraArgs = @( '-Pdev' - "-DTestResourceKey=$TestResourceKey" - "-DSuperResourceKey=$TestResourceKey" - "-DLicenseKey=$DeviceDetection" ) +if ($Depth -eq 'standard') { + $extraArgs += "-Dsurefire.excludesFile=$PSScriptRoot/standard-excludes.txt" +} +./java/run-unit-tests.ps1 -RepoName $RepoName -Name $Name -ExtraArgs $extraArgs exit $LASTEXITCODE diff --git a/ci/select-options.Tests.ps1 b/ci/select-options.Tests.ps1 new file mode 100644 index 000000000..e535a78a4 --- /dev/null +++ b/ci/select-options.Tests.ps1 @@ -0,0 +1,67 @@ +# Pester 5 tests of select-options.ps1, which chooses the options.json +# entries a pull request is built on from the files it changes. + +BeforeAll { + $Script = Join-Path $PSScriptRoot 'select-options.ps1' + $Options = Get-Content -Raw (Join-Path $PSScriptRoot 'options.json') + + $Package = 'pipeline/pipeline.core/src/main/java/fiftyone/A.java' + $NativeFile = 'device-detection/device-detection.hash.engine.on-premise/' + + 'src/main/java/fiftyone/A.java' + + # The entries chosen for the changed files, in options.json order. + function Select-Entries( + [string[]]$Changed, + [string[]]$CommentOnly = @()) { + $entries = @(ConvertFrom-Json $Options -AsHashtable) + @(& $Script -Options $entries -ChangedFiles $Changed ` + -CommentOnlyFiles $CommentOnly 6>$null) + } + + function Select-Names([string[]]$Changed, [string[]]$CommentOnly = @()) { + (Select-Entries $Changed $CommentOnly | ForEach-Object { $_.Name }) -join ',' + } +} + +Describe 'select-options.ps1' { + It 'builds the standard set for a change to a package' { + Select-Names $Package | Should -Be 'Ubuntu_Java_17,Ubuntu_Java_8' + } + + It 'adds Windows and Linux ARM for a change to a native module' { + Select-Names $NativeFile | + Should -Be 'Ubuntu_Java_17,Ubuntu_Java_8,Ubuntu_ARM_Java_17,Windows_Java_17' + } + + It 'leaves a native module change to its comments to the standard set' { + Select-Names $NativeFile $NativeFile | Should -Be 'Ubuntu_Java_17,Ubuntu_Java_8' + } + + It 'adds Windows and Linux ARM at depth deep for the cxx submodule' { + $entries = Select-Entries 'cxx' + ($entries | ForEach-Object { $_.Name }) -join ',' | + Should -Be 'Ubuntu_Java_17,Ubuntu_Java_8,Ubuntu_ARM_Java_17,Windows_Java_17' + $entries | ForEach-Object { $_.Depth | Should -Be 'deep' } + } + + It 'adds JDK 11 and 21 for a pom.xml change' { + Select-Names 'pipeline/pipeline.core/pom.xml' | + Should -Be 'Ubuntu_Java_21,Ubuntu_Java_17,Ubuntu_Java_11,Ubuntu_Java_8' + } + + It 'builds every entry but macOS, at depth deep, for a change outside the package trees' { + $entries = Select-Entries 'ci/run-unit-tests.ps1' + ($entries | ForEach-Object { $_.Name }) -join ',' | Should -Be ( + 'Ubuntu_Java_21,Ubuntu_Java_17,Ubuntu_Java_11,Ubuntu_Java_8,' + + 'Ubuntu_ARM_Java_17,Windows_Java_21,Windows_Java_17,' + + 'Windows_Java_11,Windows_Java_8') + $entries | ForEach-Object { $_.Depth | Should -Be 'deep' } + } + + It 'runs the Selenium contract on Ubuntu JDK 17 alone, and no performance tests' { + $entries = Select-Entries 'ci/run-unit-tests.ps1' + ($entries | Where-Object { $_.SeleniumContract } | + ForEach-Object { $_.Name }) -join ',' | Should -Be 'Ubuntu_Java_17' + $entries | Where-Object { $_.RunPerformance } | Should -BeNullOrEmpty + } +} diff --git a/ci/select-options.ps1 b/ci/select-options.ps1 new file mode 100644 index 000000000..1fb18ac53 --- /dev/null +++ b/ci/select-options.ps1 @@ -0,0 +1,70 @@ +param( + # Every entry of options.json. + [Parameter(Mandatory)][object[]]$Options, + # The files the pull request changes, less documentation. + [string[]]$ChangedFiles, + # Those of the changed files whose change touches only comments. + [string[]]$CommentOnlyFiles +) +# Chooses the options.json entries a pull request is built on, for common-ci's +# steps/select-options.ps1. The branch itself and a release build every entry. +# +# The standard set is JDK 17 and JDK 8 on Linux, the main JDK and the oldest, +# without the three IP Intelligence tests that load the Enterprise data file, +# and with the Selenium contract on JDK 17 alone. A change in more than its +# comments to a native module or the cxx submodule adds Windows and Linux ARM, +# where the native library is built differently. A change to IP Intelligence +# or the cxx submodule runs the three tests, and a change to a pom.xml adds +# JDK 11 and 21. A change to a file outside the package trees, such as ci, +# counts for every entry. Only five macOS jobs run at once across the +# organisation, and the releases need them, so macOS is left to the branch and +# release builds, as is performance. + +$Standard = @('Ubuntu_Java_17', 'Ubuntu_Java_8') +$Contract = 'Ubuntu_Java_17' +$NativeEntries = @('Windows_Java_17', 'Ubuntu_ARM_Java_17') +$Jdk = @('Ubuntu_Java_11', 'Ubuntu_Java_21') + +$PackageTrees = @('pipeline/', 'device-detection/', 'ip-intelligence/', + 'location/', 'examples/') +$NativePaths = @('device-detection/device-detection.hash.engine.on-premise/', + 'ip-intelligence/ip-intelligence.engine.on-premise/') +$IpIntelligencePaths = @('ip-intelligence/', 'examples/ip-intelligence/') + +function Test-Under([string]$Path, [string[]]$Prefixes) { + return @($Prefixes | Where-Object { $Path.StartsWith($_) }).Count -gt 0 +} + +$code = @($ChangedFiles | Where-Object { $CommentOnlyFiles -notcontains $_ }) +$cxx = @($code | Where-Object { $_ -eq 'cxx' }).Count -gt 0 +$native = $cxx -or @($code | Where-Object { Test-Under $_ $NativePaths }).Count -gt 0 +$ipi = $cxx -or + @($code | Where-Object { Test-Under $_ $IpIntelligencePaths }).Count -gt 0 +$poms = @($code | Where-Object { $_ -match '(^|/)pom\.xml$' }).Count -gt 0 +$other = @($code | Where-Object { + $_ -ne 'cxx' -and $_ -notmatch '(^|/)pom\.xml$' -and + !(Test-Under $_ $PackageTrees) +}).Count -gt 0 +Write-Host ("Native: $native. IP Intelligence: $ipi. pom.xml: $poms. " + + "Outside the package trees: $other.") + +foreach ($entry in $Options) { + $name = $entry.Name + if ($entry.Image -like 'macos*') { + Write-Host "$name is left to the branch and release builds" + continue + } + $why = if ($Standard -contains $name) { 'the standard set' } + elseif ($other) { 'a change outside the package trees' } + elseif ($native -and $NativeEntries -contains $name) { 'a native change' } + elseif ($poms -and $Jdk -contains $name) { 'a pom.xml change' } + if (!$why) { + Write-Host "$name is not needed for this change" + continue + } + $entry.Depth = ($ipi -or $other) ? 'deep' : 'standard' + $entry.SeleniumContract = $name -eq $Contract + $entry.RunPerformance = $false + Write-Host "$name runs for $why, at depth $($entry.Depth)" + $entry +} diff --git a/ci/setup-environment.ps1 b/ci/setup-environment.ps1 index ac900bbd4..23e2d11a9 100644 --- a/ci/setup-environment.ps1 +++ b/ci/setup-environment.ps1 @@ -1,6 +1,8 @@ param( [Parameter(Mandatory)][string]$RepoName, - [Parameter(Mandatory)][string]$JavaSDKEnvVar + [Parameter(Mandatory)][string]$JavaSDKEnvVar, + # standard, from select-options.ps1, leaves out the tests WMIC is for. + [string]$Depth ) $ErrorActionPreference = "Stop" $PSNativeCommandUseErrorActionPreference = $true @@ -11,7 +13,9 @@ $PSNativeCommandUseErrorActionPreference = $true # Without WMIC a forked test JVM on Windows can end itself part way through # the long running data file tests. The script explains why. -& "$PSScriptRoot/install-wmic.ps1" +if ($Depth -ne 'standard') { + & "$PSScriptRoot/install-wmic.ps1" +} # The native builds run PreBuild.sh, which has to be executable whatever the # checkout did to its mode. diff --git a/ci/standard-excludes.txt b/ci/standard-excludes.txt new file mode 100644 index 000000000..03d3b4b92 --- /dev/null +++ b/ci/standard-excludes.txt @@ -0,0 +1,8 @@ +# Tests that the standard set of a pull request leaves out, for +# run-unit-tests.ps1. Each loads the multi-gigabyte IP Intelligence Enterprise +# data file. +**/fiftyone/ipintelligence/IPIntelligenceTests.* +**/fiftyone/ipintelligence/examples/console/MetadataOnPremTest.* +**/fiftyone/ipintelligence/examples/console/CompareOnPremTest.* +# Surefire's own default, which a file of excludes replaces. +**/*$* diff --git a/ci/start-chrome-once.ps1 b/ci/start-chrome-once.ps1 new file mode 100644 index 000000000..c8e5733f5 --- /dev/null +++ b/ci/start-chrome-once.ps1 @@ -0,0 +1,245 @@ +# Starts Chrome and its driver once, waits for them to finish and prints how +# long each took. The first start of Chrome on a fresh runner takes many times +# longer than the starts after it, so the Selenium contract tests run this +# first and that cost falls outside the limits of their tests. A start that +# takes longer than the limit is stopped, and one that exits with an error is +# reported with its error output, both as warnings, and the tests then run as +# they would have without it. +# +# Starting Chrome directly does not take away all of the first test's extra +# time, so it then also runs one session through the driver, as each test +# does: the driver started and ready, a headless session, a blank page and the +# session ended, each part timed. +param( + # The longest each first start is given. + [int]$LimitSeconds = 300 +) +$ErrorActionPreference = "Stop" + +# The first of the names in a folder on the path, taking each folder in turn, +# as the contract tests search. +function Find-OnPath { + param([string[]]$Names) + foreach ($folder in ($env:PATH -split [System.IO.Path]::PathSeparator)) { + if (-not $folder) { + continue + } + foreach ($name in $Names) { + $candidate = Join-Path $folder $name + if (Test-Path $candidate -PathType Leaf) { + return $candidate + } + } + } + return $null +} + +# The browser the contract tests start, being the file CHROME_BIN names, then +# the first browser on the path. When neither gives one the tests leave the +# driver to find Chrome where it installs itself, so that place is used here. +function Find-Chrome { + if ($env:CHROME_BIN -and (Test-Path $env:CHROME_BIN -PathType Leaf)) { + return $env:CHROME_BIN + } + $onPath = Find-OnPath 'google-chrome', 'google-chrome-stable', 'chromium-browser', 'chromium', 'chrome.exe' + if ($onPath) { + return $onPath + } + $installed = if ($IsMacOS) { + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' + } elseif ($IsWindows) { + Join-Path ([Environment]::GetFolderPath('ProgramFiles')) 'Google\Chrome\Application\chrome.exe' + } + if ($installed -and (Test-Path $installed -PathType Leaf)) { + return $installed + } + return $null +} + +# The driver the contract tests start, being the file CHROMEWEBDRIVER names or +# the driver in the folder it names, then the first driver on the path. +function Find-ChromeDriver { + $names = 'chromedriver', 'chromedriver.exe' + $configured = $env:CHROMEWEBDRIVER + if ($configured) { + if (Test-Path $configured -PathType Leaf) { + return $configured + } + foreach ($name in $names) { + $candidate = Join-Path $configured $name + if (Test-Path $candidate -PathType Leaf) { + return $candidate + } + } + } + return Find-OnPath $names +} + +# Runs a program, stopping it after the limit. Gives the seconds it took, its +# exit code, or -1 when it was stopped, and the start of its error output. +function Measure-Start { + param([string]$FilePath, [string[]]$Arguments) + $output = [System.IO.Path]::GetTempFileName() + $errors = [System.IO.Path]::GetTempFileName() + try { + $watch = [System.Diagnostics.Stopwatch]::StartNew() + $process = Start-Process -FilePath $FilePath -ArgumentList $Arguments -PassThru -NoNewWindow ` + -RedirectStandardOutput $output -RedirectStandardError $errors + # Reading the handle now keeps the exit code readable after the exit. + $null = $process.Handle + $exitCode = -1 + if ($process.WaitForExit($LimitSeconds * 1000)) { + $process.WaitForExit() + $exitCode = $process.ExitCode + } else { + Stop-Process -Id $process.Id -Force -ErrorAction SilentlyContinue + } + return [pscustomobject]@{ + Seconds = [math]::Round($watch.Elapsed.TotalSeconds, 1) + ExitCode = $exitCode + Errors = ((Get-Content $errors -TotalCount 5 -ErrorAction SilentlyContinue) -join ' ') + } + } finally { + Remove-Item $output, $errors -Force -ErrorAction SilentlyContinue + } +} + +# Prints the result of one start, as a warning unless it exited cleanly. +function Write-Start { + param([string]$What, $Result) + if ($Result.ExitCode -eq 0) { + Write-Host "Started $What once before the Selenium contract tests in $($Result.Seconds) s" + } elseif ($Result.ExitCode -eq -1) { + Write-Host "::warning title=Chrome slow to start::$What did not finish its first start within $LimitSeconds s, so it was stopped." + } else { + Write-Host "::warning title=Chrome did not start::$What exited with code $($Result.ExitCode) after $($Result.Seconds) s. $($Result.Errors)" + } +} + +$driver = Find-ChromeDriver +if ($driver) { + Write-Start -What "the Chrome driver $driver" -Result (Measure-Start -FilePath $driver -Arguments '--version') +} + +$chrome = Find-Chrome +if (-not $chrome) { + Write-Host "::warning title=Chrome not started::No Chrome was found to start before the Selenium contract tests." + return +} +$userDataDir = Join-Path ([System.IO.Path]::GetTempPath()) "chrome-first-start-$PID" +try { + Write-Start -What "Chrome $chrome" -Result (Measure-Start -FilePath $chrome -Arguments @( + '--headless', '--disable-gpu', '--no-first-run', '--use-mock-keychain', + "--user-data-dir=$userDataDir", '--dump-dom', 'about:blank')) +} finally { + Remove-Item -Recurse -Force $userDataDir -ErrorAction SilentlyContinue +} + +# The browser the contract tests name to the driver, being the file CHROME_BIN +# names, then the first browser on the path. With neither, the tests name none +# and the driver finds Chrome itself. +function Find-ChromeBinary { + if ($env:CHROME_BIN -and (Test-Path $env:CHROME_BIN -PathType Leaf)) { + return $env:CHROME_BIN + } + return Find-OnPath 'google-chrome', 'google-chrome-stable', 'chromium-browser', 'chromium', 'chrome.exe' +} + +# A free port on the loopback address for the driver to listen on. +function Get-FreePort { + $listener = [System.Net.Sockets.TcpListener]::new([System.Net.IPAddress]::Loopback, 0) + $listener.Start() + try { + return $listener.LocalEndpoint.Port + } finally { + $listener.Stop() + } +} + +# Sends one WebDriver command and gives the seconds it took and its value. +function Invoke-DriverCommand { + param([string]$Method, [string]$Uri, $Body) + $arguments = @{ Method = $Method; Uri = $Uri; TimeoutSec = $LimitSeconds } + if ($null -ne $Body) { + $arguments.Body = ($Body | ConvertTo-Json -Depth 6 -Compress) + $arguments.ContentType = 'application/json' + } + $watch = [System.Diagnostics.Stopwatch]::StartNew() + $response = Invoke-RestMethod @arguments + return [pscustomobject]@{ + Seconds = [math]::Round($watch.Elapsed.TotalSeconds, 1) + Value = $response.value + } +} + +# Starts the driver, waits until it says it is ready, opens a headless session +# as the first contract test does, loads a blank page, ends the session and +# stops the driver, printing how long each part took. A part that fails is a +# warning, and the driver and the browser's profile are always cleaned up. +function Measure-DriverSession { + param([string]$Driver, [string]$Binary) + $base = "http://127.0.0.1:$(Get-FreePort)" + $userDataDir = Join-Path ([System.IO.Path]::GetTempPath()) "chrome-driver-session-$PID" + $output = [System.IO.Path]::GetTempFileName() + $errors = [System.IO.Path]::GetTempFileName() + $process = $null + $session = $null + try { + $watch = [System.Diagnostics.Stopwatch]::StartNew() + $process = Start-Process -FilePath $Driver -ArgumentList "--port=$(([uri]$base).Port)" ` + -PassThru -NoNewWindow -RedirectStandardOutput $output -RedirectStandardError $errors + $ready = $false + while (-not $ready -and -not $process.HasExited -and + $watch.Elapsed.TotalSeconds -lt $LimitSeconds) { + try { + $ready = [bool](Invoke-RestMethod -Uri "$base/status" -TimeoutSec 5).value.ready + } catch { + Start-Sleep -Milliseconds 250 + } + } + $readySeconds = [math]::Round($watch.Elapsed.TotalSeconds, 1) + if (-not $ready) { + Write-Host "::warning title=Chrome driver not ready::The Chrome driver $Driver was not ready after $readySeconds s. $((Get-Content $errors -TotalCount 5 -ErrorAction SilentlyContinue) -join ' ')" + return + } + $chromeOptions = @{ args = @('--headless', "--user-data-dir=$userDataDir") } + if ($Binary) { + $chromeOptions.binary = $Binary + } + $created = Invoke-DriverCommand -Method Post -Uri "$base/session" -Body @{ + capabilities = @{ + alwaysMatch = @{ + browserName = 'chrome' + acceptInsecureCerts = $true + 'goog:chromeOptions' = $chromeOptions + } + } + } + $session = $created.Value.sessionId + $page = Invoke-DriverCommand -Method Post -Uri "$base/session/$session/url" -Body @{ url = 'about:blank' } + $ended = Invoke-DriverCommand -Method Delete -Uri "$base/session/$session" + $session = $null + Write-Host ("Ran one Chrome driver session before the Selenium contract tests. " + + "The driver was ready in $readySeconds s, the session started in $($created.Seconds) s, " + + "a blank page loaded in $($page.Seconds) s and the session ended in $($ended.Seconds) s") + } catch { + Write-Host "::warning title=Chrome driver session failed::$($_.Exception.Message)" + } finally { + if ($session) { + try { + Invoke-RestMethod -Method Delete -Uri "$base/session/$session" -TimeoutSec 30 | Out-Null + } catch { + Write-Host "The session $session could not be ended: $($_.Exception.Message)" + } + } + if ($process -and -not $process.HasExited) { + Stop-Process -Id $process.Id -Force -ErrorAction SilentlyContinue + } + Remove-Item $output, $errors -Force -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force $userDataDir -ErrorAction SilentlyContinue + } +} + +if ($driver) { + Measure-DriverSession -Driver $driver -Binary (Find-ChromeBinary) +} diff --git a/ci/update-packages.ps1 b/ci/update-packages.ps1 index adec719d1..5bc798f62 100644 --- a/ci/update-packages.ps1 +++ b/ci/update-packages.ps1 @@ -13,7 +13,7 @@ $PSNativeCommandUseErrorActionPreference = $true # pipeline.version is left alone. The libraries set it to ${project.version} # so that they build against the Pipeline API in this repository, and it must # not be replaced by the latest published version. -foreach ($package in $Packages) { +foreach ($package in $Trees) { $allowSnapshots = (-not $package.Publish).ToString().ToLower() Write-Host "Updating dependencies of '$($package.Path)'. Patch version only. Snapshots allowed: $allowSnapshots." Push-Location "$RepoName/$($package.Path)" diff --git a/context7.json b/context7.json new file mode 100644 index 000000000..50929af0c --- /dev/null +++ b/context7.json @@ -0,0 +1,47 @@ +{ + "$schema": "https://context7.com/schema/context7.json", + "url": "https://context7.com/51degrees/51degrees-java", + "public_key": "pk_NN7JecmrkO5zrSbbVRsy9", + "projectTitle": "51Degrees Device Detection, IP Intelligence, Human Confidence, and Identifiers for Java", + "description": "Maven artifacts com.51degrees:device-detection and com.51degrees:ip-intelligence. new DeviceDetectionPipelineBuilder().useOnPremise(dataFile, false) or .useCloud(resourceKey), then build(). In try-with-resources, data.addEvidence(\"header.user-agent\", ua), data.process(), and data.get(DeviceData.class). IPIntelligencePipelineBuilder covers IP Intelligence. Web apps use the PipelineFilter servlet filter or Spring MVC FiftyOneInterceptor. DidClient in pipeline.did handles 51Did.", + "rules": [ + "This library is copyright of 51 Degrees Mobile Experts Limited and licensed under the European Union Public Licence (EUPL) v1.2, https://opensource.org/licenses/EUPL-1.2.", + "Code copied or adapted from this library must keep its copyright notices and the notices that refer to the EUPL, and must come with a copy of the EUPL, as Article 5 of the EUPL requires.", + "Code from this library that has been changed must carry a prominent notice saying that it was modified and the date it was modified, as Article 5 of the EUPL requires.", + "In a network application, the attribution notices that Article 5 of the EUPL requires may be given in the application's end user terms under an appropriate heading." + ], + "excludeFolders": [ + ".github", + "ci", + "cxx", + ".mvn", + "test", + "pipeline/pipeline.javascriptbuilder/src/main/resources/fiftyone/pipeline/javascriptbuilder/templates", + "pipeline/owid-java", + "pipeline/pipeline.core/configuration-validation", + "archive", + "archived", + "old", + "deprecated", + "legacy", + "previous", + "outdated", + "superseded", + "benchmark", + "benchmarks", + "benches" + ], + "excludeFiles": [ + "CHANGELOG.md", + "changelog.md", + "CHANGELOG.mdx", + "changelog.mdx", + "LICENSE.md", + "license.md", + "CODE_OF_CONDUCT.md", + "code_of_conduct.md", + "NEWS.md", + "Gaps-vs-spec.md", + "todo.txt" + ] +} diff --git a/cxx b/cxx index 7992af701..178371ac8 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit 7992af7010b7654c115510938a2f7626204bf936 +Subproject commit 178371ac8d5dec71e625fcdcb833d1dcd1ce3480 diff --git a/device-detection/README.md b/device-detection/README.md index f1e0a9e9e..73a060856 100644 --- a/device-detection/README.md +++ b/device-detection/README.md @@ -22,7 +22,7 @@ The Java API can either use our cloud service to get its data or it can use a lo You will require [resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud) to use the Cloud API, as described on our website. Get resource keys from -our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud) on +our [configuration page](https://51degrees.com/configure/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud) on how to use this. #### On-Premise @@ -230,10 +230,13 @@ You will need [resource keys](https://51degrees.com/documentation/_services__clo To verify the code: ``` -mvn clean test -DTestResourceKey=[Resource Key] +mvn clean test -D_51DEGREES_RESOURCE_KEY=[Resource Key] ``` -For tests and examples that require a license key add the following option: -- `-DLicenseKey=[License Key]` +The key can be set as the `_51DEGREES_RESOURCE_KEY` environment variable +instead. The TAC and native model examples read a key with every property +from `_51DEGREES_RESOURCE_KEY_SUPER`. For tests and examples that require a +license key, set `_51DEGREES_LICENSE_KEY` or add the following option: +- `-D_51DEGREES_LICENSE_KEY=[License Key]` ## Projects diff --git a/device-detection/device-detection.cloud/pom.xml b/device-detection/device-detection.cloud/pom.xml index ddfbaf069..825e1d517 100644 --- a/device-detection/device-detection.cloud/pom.xml +++ b/device-detection/device-detection.cloud/pom.xml @@ -11,7 +11,7 @@ device-detection.cloud 51Degrees :: Device Detection :: Cloud - Cloud-based device detection for Java that uses the 51Degrees cloud service via the Pipeline API. Parses HTTP headers to identify device hardware, operating systems, browsers, and crawlers. + Device Detection and User-Agent parser for Java, using the 51Degrees cloud service. Detects the device type and model, browser, operating system, and crawlers from the User-Agent and User-Agent Client Hints. Needs a Resource Key that says which properties to return. Built on the 51Degrees Pipeline API. An alternative to UAParser.js, DeviceAtlas, and WURFL. https://github.com/51Degrees/51degrees-java diff --git a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/data/MultiDeviceDataCloud.java b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/data/MultiDeviceDataCloud.java index 2a41ed429..2abcc8a85 100644 --- a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/data/MultiDeviceDataCloud.java +++ b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/data/MultiDeviceDataCloud.java @@ -85,8 +85,12 @@ public List getProfiles() { return getDeviceList(); } + /** + * @throws IllegalStateException if the Flow Element has finished + */ @Override public void addProfile(DeviceData profile) { + checkWriting(); getDeviceList().add(profile); } diff --git a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java index 400ca1dad..06a25cf3b 100644 --- a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java +++ b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java @@ -41,8 +41,17 @@ import org.json.JSONObject; import org.slf4j.Logger; +import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.NoCloudResponse; + +import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.math.BigDecimal; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.TreeMap; @@ -55,6 +64,13 @@ */ public class DeviceDetectionCloudEngine extends CloudAspectEngineBase { + /** + * The type of value each typed accessor of the device data gives, keyed + * by the property name in lower case, so that a value the metadata does + * not describe can be given as the type its accessor declares. + */ + private static final Map DECLARED_TYPES = declaredTypes(); + volatile List aspectProperties; private String dataSourceTier; CloudRequestEngine cloudRequestEngine; @@ -135,8 +151,19 @@ protected void processEngine(FlowData data, DeviceDataCloud aspectData) { else { CloudRequestData requestData = data.getFromElement(cloudRequestEngine); - String json = ""; - json = requestData.getJsonResponse(); + String json = getJsonResponse( + data, + aspectData, + requestData, + cloudRequestEngine); + + // A failed cloud request leaves no response. The Cloud Request + // Engine reports why, so this engine adds no error of its own. + if (json == null || json.isEmpty()) { + logger.warn(String.format( + NoCloudResponse, getClass().getSimpleName())); + return; + } // Extract data from json to the aspectData instance. JSONObject jsonObj = new JSONObject(json); @@ -146,6 +173,12 @@ protected void processEngine(FlowData data, DeviceDataCloud aspectData) { new TreeMap<>(String.CASE_INSENSITIVE_ORDER); for (AspectPropertyMetaData property : getProperties()) { + // A property the response leaves out has no entry, so + // reading it throws a PropertyMissingException. + if (deviceObj.has( + property.getName().toLowerCase(Locale.ROOT)) == false) { + continue; + } String type = property.getType().getSimpleName(); switch(type) { case ("List"): @@ -186,6 +219,18 @@ protected void processEngine(FlowData data, DeviceDataCloud aspectData) { } } + // A value the metadata does not describe, as when the metadata + // is empty before discovery has completed, takes its type from + // the value itself. + for (String key : deviceObj.keySet()) { + if (key.endsWith("nullreason") == false && + deviceMap.containsKey(key) == false) { + deviceMap.put( + key, + getInferredAspectPropertyValue(deviceObj, key)); + } + } + // Add no value messages for any properties which have not yet been // populated. for (String key : deviceObj.keySet()) { @@ -272,7 +317,7 @@ private boolean loadAspectProperties(CloudRequestEngine engine) { private AspectPropertyValue getIntegerAspectPropertyValue( JSONObject deviceObj, AspectPropertyMetaData property) { - String key = property.getName().toLowerCase(); + String key = property.getName().toLowerCase(Locale.ROOT); AspectPropertyValue intValue = new AspectPropertyValueDefault<>(); if(deviceObj.isNull(key)){ intValue.setNoValueMessage(getNoValueReason(deviceObj, key)); @@ -284,7 +329,7 @@ private AspectPropertyValue getIntegerAspectPropertyValue( } private AspectPropertyValue getDoubleAspectPropertyValue(JSONObject deviceObj, AspectPropertyMetaData property) { - String key = property.getName().toLowerCase(); + String key = property.getName().toLowerCase(Locale.ROOT); AspectPropertyValue doubleValue = new AspectPropertyValueDefault<>(); if(deviceObj.isNull(key)){ doubleValue.setNoValueMessage(getNoValueReason(deviceObj, key)); @@ -306,7 +351,7 @@ private AspectPropertyValue getDoubleAspectPropertyValue(JSONObject devi private AspectPropertyValue getBooleanAspectPropertyValue( JSONObject deviceObj, AspectPropertyMetaData property) { - String key = property.getName().toLowerCase(); + String key = property.getName().toLowerCase(Locale.ROOT); AspectPropertyValue booleanValue = new AspectPropertyValueDefault<>(); if(deviceObj.isNull(key)){ booleanValue.setNoValueMessage(getNoValueReason(deviceObj, key)); @@ -328,7 +373,7 @@ private AspectPropertyValue getBooleanAspectPropertyValue( private AspectPropertyValue> getListAspectPropertyValue( JSONObject deviceObj, AspectPropertyMetaData property){ - String key = property.getName().toLowerCase(); + String key = property.getName().toLowerCase(Locale.ROOT); AspectPropertyValue> listValue = new AspectPropertyValueDefault<>(); if(deviceObj.isNull(key)) { @@ -357,7 +402,7 @@ private AspectPropertyValue> getListAspectPropertyValue( private AspectPropertyValue getJavaScriptAspectPropertyValue( JSONObject deviceObj, AspectPropertyMetaData property){ - String key = property.getName().toLowerCase(); + String key = property.getName().toLowerCase(Locale.ROOT); AspectPropertyValue jsValue = new AspectPropertyValueDefault<>(); if(deviceObj.isNull(key)){ jsValue.setNoValueMessage(getNoValueReason(deviceObj, key)); @@ -378,7 +423,7 @@ private AspectPropertyValue getJavaScriptAspectPropertyValue( private AspectPropertyValue getStringAspectPropertyValue( JSONObject deviceObj, AspectPropertyMetaData property){ - String key = property.getName().toLowerCase(); + String key = property.getName().toLowerCase(Locale.ROOT); AspectPropertyValue stringValue = new AspectPropertyValueDefault<>(); if(deviceObj.isNull(key)){ stringValue.setNoValueMessage(getNoValueReason(deviceObj, key)); @@ -389,6 +434,100 @@ private AspectPropertyValue getStringAspectPropertyValue( return stringValue; } + /** + * Get a value from the cloud engine's JSON response that the metadata + * does not describe, with its type taken from the value: a string, a + * boolean, a number, a list of strings, or an object as a map. A + * number is an Integer where the typed accessor of + * {@link DeviceDataCloud} declares one and the number is a whole + * number an Integer can hold, and a Double where the accessor declares + * one or the number has a decimal point. A null is no value, with the + * reason the response gives. + * @param deviceObj to get the value from + * @param key to get the value of + * @return {@link AspectPropertyValue} with the value, or the reason for + * the value not being present + */ + private AspectPropertyValue getInferredAspectPropertyValue( + JSONObject deviceObj, + String key) { + Object value = deviceObj.opt(key); + if (value == null || value == JSONObject.NULL) { + AspectPropertyValue noValue = + new AspectPropertyValueDefault<>(); + noValue.setNoValueMessage(getNoValueReason(deviceObj, key)); + return noValue; + } + if (value instanceof JSONArray) { + JSONArray jsonArray = (JSONArray) value; + List strings = new ArrayList<>(jsonArray.length()); + for (Object item : jsonArray.toList()) { + strings.add(Objects.toString(item, null)); + } + return new AspectPropertyValueDefault<>(strings); + } + if (value instanceof JSONObject) { + return new AspectPropertyValueDefault<>( + ((JSONObject) value).toMap()); + } + if (value instanceof Number) { + Number number = (Number) value; + Type declared = DECLARED_TYPES.get(key.toLowerCase(Locale.ROOT)); + if (Integer.class.equals(declared) && isWholeInteger(number)) { + return new AspectPropertyValueDefault<>(number.intValue()); + } + if (Double.class.equals(declared) || + value instanceof BigDecimal || + value instanceof Double || + value instanceof Float) { + return new AspectPropertyValueDefault<>( + number.doubleValue()); + } + } + return new AspectPropertyValueDefault<>(value); + } + + /** + * True when the number is a whole number an Integer can hold, such as + * 24 or 24.0, and false for 24.5 or a number beyond an Integer's range. + * @param number the number + * @return true if the number can be an Integer without changing it + */ + private static boolean isWholeInteger(Number number) { + double value = number.doubleValue(); + return value == Math.rint(value) && + value >= Integer.MIN_VALUE && + value <= Integer.MAX_VALUE; + } + + /** + * The type of value each typed accessor of {@link DeviceDataCloud} + * gives, being the type argument of the {@link AspectPropertyValue} it + * returns, keyed by the property name in lower case. + * @return the declared types + */ + private static Map declaredTypes() { + Map types = new HashMap<>(); + for (Method method : DeviceDataCloud.class.getMethods()) { + Type returned = method.getGenericReturnType(); + if (method.getParameterCount() == 0 && + method.getName().startsWith("get") && + AspectPropertyValue.class.equals(method.getReturnType()) && + returned instanceof ParameterizedType) { + types.put( + method.getName().substring(3).toLowerCase(Locale.ROOT), + ((ParameterizedType) returned).getActualTypeArguments()[0]); + } + } + return Collections.unmodifiableMap(types); + } + + /** + * The message for a property with no value when the response gives no + * reason. + */ + private static final String UNKNOWN_NULL_REASON = "Unknown"; + /** * Get the reason for the value of a property not being present in the cloud * engine's JSON response. @@ -398,7 +537,9 @@ private AspectPropertyValue getStringAspectPropertyValue( */ private String getNoValueReason(JSONObject deviceObj, String key){ Object reason = tryToGet(deviceObj, key + "nullreason"); - return reason == null ? null : reason.toString(); + return reason == null || reason == JSONObject.NULL ? + UNKNOWN_NULL_REASON : + reason.toString(); } /** diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TestsBase.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TestsBase.java index 44d233d6a..f307376c2 100644 --- a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TestsBase.java +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TestsBase.java @@ -25,7 +25,7 @@ import fiftyone.devicedetection.shared.testhelpers.UserAgentGenerator; import fiftyone.pipeline.util.FileFinder; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_PATH; +import static fiftyone.devicedetection.shared.testhelpers.TestFileUtils.UA_FILE_PATH; public class TestsBase { diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TimedHttpClient.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TimedHttpClient.java new file mode 100644 index 000000000..a6ce76f07 --- /dev/null +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/TimedHttpClient.java @@ -0,0 +1,81 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.cloud; + +import fiftyone.pipeline.engines.services.HttpClientDefault; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.util.Map; + +/** + * Sends requests as the default client does, and prints how long each one + * to the live cloud service took, so each run records the measurement the + * tests' cloud timeout rests on. Only the path is printed, never the query, + * and a path's last part ending in ".json" is replaced, as it is the + * Resource Key. + */ +class TimedHttpClient extends HttpClientDefault { + + @Override + public String postData( + HttpURLConnection connection, + Map headers, + byte[] data) throws IOException { + long start = System.currentTimeMillis(); + String outcome = "failed"; + try { + String reply = super.postData(connection, headers, data); + outcome = "answered"; + return reply; + } finally { + print(connection, outcome, start); + } + } + + @Override + public String getResponseString( + HttpURLConnection connection, + Map headers) throws IOException { + long start = System.currentTimeMillis(); + String outcome = "failed"; + try { + String reply = super.getResponseString(connection, headers); + outcome = "answered"; + return reply; + } finally { + print(connection, outcome, start); + } + } + + private static void print( + HttpURLConnection connection, + String outcome, + long start) { + System.out.println( + "Cloud request " + + connection.getURL().getPath() + .replaceAll("[^/]+\\.json$", ".json") + " " + + outcome + " in " + (System.currentTimeMillis() - start) + " ms"); + } +} diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/WrapperCloud.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/WrapperCloud.java index 420d074c9..324a81db3 100644 --- a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/WrapperCloud.java +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/WrapperCloud.java @@ -24,7 +24,8 @@ import fiftyone.devicedetection.cloud.flowelements.DeviceDetectionCloudEngine; import fiftyone.devicedetection.cloud.flowelements.DeviceDetectionCloudEngineBuilder; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; +import fiftyone.devicedetection.shared.testhelpers.Constants; +import fiftyone.devicedetection.shared.testhelpers.TestKeyUtils; import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngineBuilder; import fiftyone.pipeline.core.flowelements.Pipeline; @@ -37,6 +38,8 @@ import org.slf4j.ILoggerFactory; import org.slf4j.LoggerFactory; +import static org.junit.Assume.assumeFalse; + public class WrapperCloud implements Closeable { protected static final ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory(); @@ -45,9 +48,27 @@ public class WrapperCloud implements Closeable { public DeviceDetectionCloudEngine deviceDetectionCloudEngine; public WrapperCloud() throws Exception { + String resourceKey = + TestKeyUtils.getNamedKey(Constants.RESOURCE_KEY_ENV_VAR); + assumeFalse("These tests call the live 51Degrees cloud service, so " + + "they need a resource key, and they are skipped because " + + Constants.RESOURCE_KEY_ENV_VAR + " is not set as an " + + "environment variable or as a system property.", + TestKeyUtils.isInvalidKey(resourceKey)); cloudRequestEngine = - new CloudRequestEngineBuilder(loggerFactory) - .setResourceKey(KeyUtils.getNamedKey("TestResourceKey")) + new CloudRequestEngineBuilder(loggerFactory, new TimedHttpClient()) + .setResourceKey(resourceKey) + // The live cloud service is shared with every CI run. In run + // 44881631156 its slowest answer was 0.77 s, over 540 + // requests on all 15 Build and Test jobs (TimedHttpClient + // prints each one). In java #190's release build (run + // 44702767584, about 08:38 UTC on 20 November 2026) device + // detection's ValueCloudTests got "Read timed out" past the + // 2 second default on two jobs. How long those answers took + // was not measured. 15 seconds is about 20 times the slowest + // answer measured. These tests check what the service + // answers, not how fast. + .setTimeOutSeconds(15) .build(); deviceDetectionCloudEngine = new DeviceDetectionCloudEngineBuilder(loggerFactory) .build(); diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/CloudRequestFailedTests.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/CloudRequestFailedTests.java new file mode 100644 index 000000000..5616e6187 --- /dev/null +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/CloudRequestFailedTests.java @@ -0,0 +1,185 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.cloud.flowelements; + +import fiftyone.devicedetection.cloud.data.DeviceDataCloud; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * When the Cloud Request Engine's request for the Flow Data failed, a + * property of the Device Detection cloud engine is missing with the + * CloudRequestFailed reason, rather than a reason about the resource key. + */ +public class CloudRequestFailedTests { + + private static final String CLOUD_REQUEST_FAILED = + "Property 'hardwarevendor' not found in data for element 'device'. " + + "This is because an upstream cloud request failed, so no data was " + + "available to populate the property."; + + private ILoggerFactory loggerFactory; + + private CloudRequestEngine requestEngine; + + @Before + public void init() { + Logger logger = mock(Logger.class); + loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + } + + @Test + public void RequestFailedWithNoResponse_CloudRequestFailed() + throws Exception { + assertCloudRequestFailed(null); + } + + @Test + public void RequestFailedWithAnErrorResponse_CloudRequestFailed() + throws Exception { + assertCloudRequestFailed("{ \"errors\": [ \"an error\" ] }"); + } + + @Test + public void RequestSucceeded_ReasonIsAboutTheResourceKey() + throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process( + engine, + "{ \"device\": { \"ismobile\": true } }", + Collections.emptyList()); + assertMissing( + "PropertyNotAccessibleWithResourceKey", + null, + data); + } + } + + private void assertCloudRequestFailed(String response) throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process( + engine, + response, + Collections.singletonList(new FlowError.Default( + new Exception("The cloud request failed."), + requestEngine))); + assertMissing("CloudRequestFailed", CLOUD_REQUEST_FAILED, data); + } + } + + private static void assertMissing( + String reason, + String message, + DeviceDataCloud data) { + try { + data.getHardwareVendor(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + assertEquals(e.getMessage(), reason, e.getReason().name()); + if (message != null) { + assertEquals(message, e.getMessage()); + } + } + } + + /** + * An engine whose Cloud Request Engine reports the device product, + * holding IsMobile alone. + */ + // thenReturn has a varargs overload, so a generic Map is an unchecked array. + @SuppressWarnings("unchecked") + private DeviceDetectionCloudEngine buildEngine() throws Exception { + DeviceDetectionCloudEngine engine = + new DeviceDetectionCloudEngineBuilder(loggerFactory).build(); + + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("ismobile", "Boolean", "cat", null)); + Map publicProperties = new HashMap<>(); + publicProperties.put("device", product); + + requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(publicProperties); + + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + engine.addPipeline(pipeline); + return engine; + } + + private DeviceDataCloud process( + DeviceDetectionCloudEngine engine, + String response, + Collection errors) throws Exception { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn(response); + when(cloudData.getProcessStarted()).thenReturn(true); + + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + when(flowData.getErrors()).thenReturn(errors); + + DeviceDataCloud device = new DeviceDataCloudInternal( + loggerFactory.getLogger(DeviceDataCloud.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + try { + engine.processEngine(flowData, device); + } catch (RuntimeException e) { + // A failed request's response may not be readable. The + // property is read afterwards either way, as it is through a + // Pipeline that suppresses process exceptions. + } + return device; + } +} diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/CloudRequestFailureRecordedTests.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/CloudRequestFailureRecordedTests.java new file mode 100644 index 000000000..ecfc20266 --- /dev/null +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/CloudRequestFailureRecordedTests.java @@ -0,0 +1,156 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.cloud.flowelements; + +import fiftyone.devicedetection.cloud.data.DeviceDataCloud; +import fiftyone.pipeline.cloudrequestengine.CloudRequestException; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngineBuilder; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.HttpClient; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.stubbing.Answer; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * A failed Cloud Request Engine request that the Device Detection cloud + * engine meets while the Flow Data is processed is recorded once, as the + * Cloud Request Engine's error and not the Device Detection engine's, and a + * property is then missing with the CloudRequestFailed reason. + * @see Specification + */ +public class CloudRequestFailureRecordedTests { + + private static final String PROPERTIES = + "{'Products': {'device': {'DataTier': 'tier', 'Properties': " + + "[{'Name': 'HardwareVendor', 'Type': 'String', " + + "'Category': 'Device'}]}}}"; + + private static final String EVIDENCE_KEYS = "['query.User-Agent']"; + + private static final String FAILED = + "{\"errors\":[\"The request failed.\"]}"; + + /** + * A client that answers the discovery requests and fails the request + * for the Flow Data with an error response. + */ + private static HttpClient failingClient() throws Exception { + HttpClient client = mock(HttpClient.class); + when(client.connect(any(URL.class))).thenAnswer(invocation -> { + URL url = invocation.getArgument(0); + HttpURLConnection connection = mock(HttpURLConnection.class); + when(connection.getURL()).thenReturn(url); + when(connection.getResponseCode()).thenReturn(200); + when(connection.getHeaderFields()).thenReturn(new HashMap<>()); + return connection; + }); + Answer byUrl = invocation -> { + String path = ((HttpURLConnection) invocation.getArgument(0)) + .getURL().getPath(); + if (path.endsWith("properties")) { + return PROPERTIES; + } + if (path.endsWith("evidencekeys")) { + return EVIDENCE_KEYS; + } + return FAILED; + }; + when(client.getResponseString(any(HttpURLConnection.class))) + .thenAnswer(byUrl); + when(client.getResponseString( + any(HttpURLConnection.class), + ArgumentMatchers.anyMap())).thenAnswer(byUrl); + when(client.postData( + any(HttpURLConnection.class), + ArgumentMatchers.anyMap(), + any(byte[].class))).thenAnswer(byUrl); + return client; + } + + @Test + public void FailedRequest_RecordedOnceAsTheRequestEnginesError() + throws Exception { + Logger logger = mock(Logger.class); + ILoggerFactory loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + CloudRequestEngine requestEngine = + new CloudRequestEngineBuilder(loggerFactory, failingClient()) + .setResourceKey("resource_key") + .build(); + DeviceDetectionCloudEngine deviceEngine = + new DeviceDetectionCloudEngineBuilder(loggerFactory).build(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(true) + .addFlowElement(requestEngine) + .addFlowElement(deviceEngine) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + + List errors = new ArrayList<>(data.getErrors()); + assertEquals(errors.toString(), 1, errors.size()); + assertSame( + "The failure was recorded as another element's error", + requestEngine, + errors.get(0).getFlowElement()); + assertTrue( + errors.get(0).getThrowable().toString(), + errors.get(0).getThrowable() instanceof CloudRequestException); + + DeviceDataCloud device = data.getFromElement(deviceEngine); + try { + device.getHardwareVendor(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + assertEquals( + e.getMessage(), + "CloudRequestFailed", + e.getReason().name()); + } + } + } +} diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ElementDataKeyFormatTests.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ElementDataKeyFormatTests.java new file mode 100644 index 000000000..59fd7698c --- /dev/null +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ElementDataKeyFormatTests.java @@ -0,0 +1,69 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.cloud.flowelements; + +import org.junit.Test; +import org.slf4j.Logger; + +import java.util.Locale; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * The string key of each Device Detection cloud engine's data contains only + * alphanumeric characters, full stop or hyphen. + *

+ * Specification: pipeline-specification/features/access-to-results.md line 7 + * at 51Degrees/specifications 706de1f0, "Strings SHOULD only contain + * alphanumeric characters, full stop or hyphen." + */ +public class ElementDataKeyFormatTests { + + private static String[] keys() { + return new String[] { + new DeviceDetectionCloudEngine(mock(Logger.class), null) + .getElementDataKey(), + new HardwareProfileCloudEngine(mock(Logger.class), null) + .getElementDataKey() + }; + } + + @Test + public void keys_OnlyAlphanumericFullStopOrHyphen() { + for (String key : keys()) { + assertTrue(key, key.matches("[A-Za-z0-9.-]+")); + } + } + + /** + * Specification: access-to-results.md line 10, "Lower-case SHOULD be + * used." + */ + @Test + public void keys_LowerCase() { + for (String key : keys()) { + assertTrue(key, key.equals(key.toLowerCase(Locale.ROOT))); + } + } +} diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/EmptyMetadataTests.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/EmptyMetadataTests.java new file mode 100644 index 000000000..c7d121c93 --- /dev/null +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/EmptyMetadataTests.java @@ -0,0 +1,247 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.cloud.flowelements; + +import fiftyone.devicedetection.cloud.data.DeviceDataCloud; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Before the Cloud Request Engine's discovery has completed, the engine has + * no property metadata. It still reads the values in the response, taking + * the type of each from the value itself. + */ +public class EmptyMetadataTests { + + private static final String RESPONSE = + "{ 'device': { " + + "'hardwarevendor': 'Apple', " + + "'ismobile': true, " + + "'screenpixelswidth': 1170, " + + "'pixelratio': 3.5, " + + "'hardwarename': [ 'iPhone 13', 'iPhone 14' ], " + + "'hardwaremodel': null, " + + "'platformname': null, " + + "'platformnamenullreason': 'No value on this request' }, " + + "'javascriptProperties': [] }"; + + private ILoggerFactory loggerFactory; + + @Before + public void init() { + Logger logger = mock(Logger.class); + loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + } + + @Test + public void EmptyMetadata_MetadataIsAnEmptyList() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + assertTrue(engine.getProperties().isEmpty()); + assertFalse(engine.hasLoadedProperties()); + } + } + + @Test + public void EmptyMetadata_StringIsRead() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine); + assertEquals("Apple", data.getHardwareVendor().getValue()); + } + } + + @Test + public void EmptyMetadata_BooleanIsRead() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine); + assertEquals(Boolean.TRUE, data.getIsMobile().getValue()); + } + } + + @Test + public void EmptyMetadata_IntegerIsRead() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine); + assertEquals( + Integer.valueOf(1170), + data.getScreenPixelsWidth().getValue()); + } + } + + @Test + public void EmptyMetadata_DoubleIsRead() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine); + assertEquals( + Double.valueOf(3.5), + data.getPixelRatio().getValue()); + } + } + + /** + * A whole number for a property whose typed accessor gives a Double is + * read as a Double, as the cloud service can give 3 for a pixel ratio + * of 3.0. + */ + @Test + public void EmptyMetadata_WholeNumberDeclaredDouble_IsDouble() + throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine, + "{ 'device': { 'pixelratio': 3 }, " + + "'javascriptProperties': [] }"); + Object value = data.getPixelRatio().getValue(); + assertEquals(Double.class, value.getClass()); + assertEquals(3.0, (Double) value, 0); + } + } + + /** + * A decimal with no fractional part for a property whose typed + * accessor gives an Integer is read as an Integer, so the accessor + * works if the cloud service gives 24.0 for a bits per pixel of 24. + */ + @Test + public void EmptyMetadata_WholeDecimalDeclaredInteger_IsInteger() + throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine, + "{ 'device': { 'bitsperpixel': 24.0 }, " + + "'javascriptProperties': [] }"); + Object value = data.getBitsPerPixel().getValue(); + assertEquals(Integer.class, value.getClass()); + assertEquals(24, ((Integer) value).intValue()); + } + } + + /** + * A decimal with a fractional part for a property whose typed accessor + * gives an Integer is not made a whole number, because that would + * change the value the service gave. + */ + @Test + public void EmptyMetadata_FractionDeclaredInteger_IsNotRounded() + throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine, + "{ 'device': { 'bitsperpixel': 24.5 }, " + + "'javascriptProperties': [] }"); + Object value = data.get("bitsperpixel"); + assertTrue("The value is not an AspectPropertyValue.", + value instanceof AspectPropertyValue); + assertEquals(24.5, + ((Number) ((AspectPropertyValue) value).getValue()) + .doubleValue(), 0); + } + } + + @Test + public void EmptyMetadata_ListIsRead() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine); + assertEquals( + Arrays.asList("iPhone 13", "iPhone 14"), + data.getHardwareName().getValue()); + } + } + + @Test + public void EmptyMetadata_NullWithReason_HasThatReason() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine); + assertFalse(data.getPlatformName().hasValue()); + assertEquals( + "No value on this request", + data.getPlatformName().getNoValueMessage()); + } + } + + @Test + public void EmptyMetadata_NullWithoutReason_IsUnknown() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud data = process(engine); + assertFalse(data.getHardwareModel().hasValue()); + assertEquals( + "Unknown", + data.getHardwareModel().getNoValueMessage()); + } + } + + /** + * An engine whose Cloud Request Engine does not yet know the products, + * so the engine has no property metadata. + */ + private DeviceDetectionCloudEngine buildEngine() throws Exception { + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(null); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + DeviceDetectionCloudEngine engine = + new DeviceDetectionCloudEngineBuilder(loggerFactory).build(); + engine.addPipeline(pipeline); + return engine; + } + + private DeviceDataCloud process(DeviceDetectionCloudEngine engine) { + return process(engine, RESPONSE); + } + + private DeviceDataCloud process( + DeviceDetectionCloudEngine engine, + String response) { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn(response); + + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + + DeviceDataCloud device = new DeviceDataCloudInternal( + loggerFactory.getLogger(DeviceDataCloud.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + engine.processEngine(flowData, device); + return device; + } +} diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/HardwareProfileCloudEngineTests.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/HardwareProfileCloudEngineTests.java new file mode 100644 index 000000000..59f796770 --- /dev/null +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/HardwareProfileCloudEngineTests.java @@ -0,0 +1,171 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.cloud.flowelements; + +import fiftyone.devicedetection.cloud.data.MultiDeviceDataCloud; +import fiftyone.devicedetection.shared.DeviceData; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.cloudrequestengine.flowelements.PropertyKeyedCloudEngineBase; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The majority of the processing SHOULD be handled by the shared Cloud Aspect + * Engine logic + * (device-detection-specification/pipeline-elements/hardware-profile-lookup-cloud.md + * line 56 at 706de1f0). The hardware profile engine gives no processing of + * its own, and the shared logic for property keyed engines turns each + * profile in its part of the response into DeviceData. + */ +public class HardwareProfileCloudEngineTests { + + private static final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + private static HardwareProfileCloudEngine engine() throws Exception { + HardwareProfileCloudEngine engine = + new HardwareProfileCloudEngineBuilder(loggerFactory).build(); + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("profiles", "Array", "Device", Arrays.asList( + new PropertyMetaData("hardwarevendor", "String", "Device", null), + new PropertyMetaData("hardwarename", "Array", "Device", null)))); + Map products = new HashMap<>(); + products.put("hardware", product); + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(products); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + engine.addPipeline(pipeline); + return engine; + } + + private static MultiDeviceDataCloud process( + HardwareProfileCloudEngine engine, + String response) throws Exception { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn(response); + when(cloudData.getProcessStarted()).thenReturn(true); + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + MultiDeviceDataCloud data = new MultiDeviceDataCloud( + loggerFactory.getLogger(MultiDeviceDataCloud.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + // The Pipeline calls the protected processEngine of the shared base. + Method processEngine = null; + for (Class type = engine.getClass(); processEngine == null && + type != null; type = type.getSuperclass()) { + for (Method method : type.getDeclaredMethods()) { + if (method.getName().equals("processEngine") && + method.getParameterCount() == 2) { + processEngine = method; + } + } + } + processEngine.setAccessible(true); + try { + processEngine.invoke(engine, flowData, data); + } catch (InvocationTargetException e) { + throw (Exception) e.getCause(); + } + return data; + } + + /** + * The engine gives no processing of its own. + */ + @Test + public void HardwareProfile_ProcessingIsShared() { + assertTrue(PropertyKeyedCloudEngineBase.class.isAssignableFrom( + HardwareProfileCloudEngine.class)); + for (Method method : + HardwareProfileCloudEngine.class.getDeclaredMethods()) { + assertTrue("The engine processes with " + method.getName(), + method.getName().equals("processEngine") == false && + method.getName().equals("processCloudEngine") == false); + } + } + + /** + * Each profile in the engine's part of the response becomes DeviceData + * with the values the response gives. + */ + @Test + public void HardwareProfile_ProfilesAreDeviceData() throws Exception { + try (HardwareProfileCloudEngine engine = engine()) { + MultiDeviceDataCloud data = process(engine, + "{ 'hardware': { 'profiles': [" + + "{ 'hardwarevendor': 'Apple', " + + "'hardwarename': ['iPhone 12', 'iPhone 12 Pro'] }, " + + "{ 'hardwarevendor': null, " + + "'hardwarevendornullreason': 'No match', " + + "'hardwarename': [] } ] } }"); + List profiles = data.getProfiles(); + assertEquals(2, profiles.size()); + assertEquals("Apple", + profiles.get(0).getHardwareVendor().getValue()); + assertEquals(Arrays.asList("iPhone 12", "iPhone 12 Pro"), + profiles.get(0).getHardwareName().getValue()); + assertEquals(false, + profiles.get(1).getHardwareVendor().hasValue()); + assertEquals("No match", + profiles.get(1).getHardwareVendor().getNoValueMessage()); + } + } + + /** + * An empty response, after an error the Cloud Request Engine has + * reported, gives no profiles and raises no error. + */ + @Test + public void HardwareProfile_EmptyResponse_NoProfiles() throws Exception { + try (HardwareProfileCloudEngine engine = engine()) { + assertTrue(process(engine, "").getProfiles().isEmpty()); + } + } +} diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/MissingPropertyHandlingTests.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/MissingPropertyHandlingTests.java index 8e85a1ca0..60ec1c840 100644 --- a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/MissingPropertyHandlingTests.java +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/MissingPropertyHandlingTests.java @@ -39,6 +39,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Locale; import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.*; @@ -129,6 +130,59 @@ public void PropertyNotInResource() throws Exception { } } + /** + * A property whose name holds a capital I is read from the cloud's lower + * case key under a Turkish default locale, which would otherwise lower + * the I to a dotless i and look for a key the response does not have. + * + * @throws Exception + */ + @Test + public void PropertyWithCapitalI_TurkishLocale_HasValue() throws Exception { + Locale original = Locale.getDefault(); + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + try (DeviceDetectionCloudEngine engine = + new DeviceDetectionCloudEngineBuilder(loggerFactory) + .build()) { + + FlowData flowData = mock(FlowData.class); + addResponse(flowData, deviceIdJson); + DeviceDataCloud device = new DeviceDataCloudInternal( + loggerFactory.getLogger(DeviceDataCloud.class.getSimpleName()), + flowData, + engine, + mock(MissingPropertyService.class)); + engine.cloudRequestEngine = mock(CloudRequestEngine.class); + engine.aspectProperties = deviceIdProperties; + + engine.processEngine(flowData, device); + + AspectPropertyValue deviceId = device.getDeviceId(); + assertTrue(deviceId.hasValue()); + assertEquals(expectedDeviceId, deviceId.getValue()); + } finally { + Locale.setDefault(original); + } + } + + private static String expectedDeviceId = "12280-48866-24305-18092"; + + private static String deviceIdJson = + "{\n" + + " 'device': {\n" + + " 'deviceid': '" + expectedDeviceId + "'\n" + + " },\n" + + " 'javascriptProperties': []\n" + + "}\n"; + + private static List deviceIdProperties = + new ArrayList() { + private static final long serialVersionUID = 1L; + { + add(new AspectPropertyMetaDataDefault("DeviceId", null, null, String.class, null, true)); + } + }; + private static String expectedNullReason = "this is the null reason"; private static String expectedMissingPropertyReason = "this is the missing reason"; diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java index 2cd708eb6..eed6a4dd1 100644 --- a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java @@ -97,21 +97,86 @@ public void ProductMissing_ProcessLogsAndThrowsNothing() throws Exception { } } + /** + * A failed cloud request leaves no response. The engine processes + * nothing, throws nothing and logs one warning, as the Cloud Request + * Engine's error says why. + */ + @Test + public void NoResponse_ProcessesNothing() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine("device", "hardwarevendor")) { + for (String response : new String[] { null, "" }) { + DeviceDataCloud data = process(engine, response); + assertFalse(data.asKeyMap().containsKey("hardwarevendor")); + } + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 2, logged.size()); + for (String line : logged) { + assertTrue(line, line.startsWith("warn ")); + } + } + } + /** * With the Device Detection product included, a response that holds no - * part for it leaves the element data without values and throws - * nothing, as the Processing section of the cloud aspect engine - * specification says. + * part for it raises nothing while processing, and reading a property + * throws PropertyMissingException, as for any property the response + * leaves out. */ @Test - public void ProductPresent_ResponseWithoutPart_NoValues() + public void ProductPresent_ResponseWithoutPart_PropertyIsMissing() throws Exception { - try (DeviceDetectionCloudEngine engine = buildEngine("device", "hardwarevendor")) { + try (DeviceDetectionCloudEngine engine = + buildEngine("device", "hardwarevendor")) { DeviceDataCloud data = process( engine, "{ 'ip': { 'registeredname': 'Example' }, " + "'javascriptProperties': [] }"); + try { + data.getHardwareVendor(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + // The response held nothing for the product. + } + } + } + + /** + * A property the response's part leaves out has no entry, so reading it + * throws PropertyMissingException rather than giving no value. + */ + @Test + public void ProductPresent_PropertyLeftOut_PropertyIsMissing() + throws Exception { + try (DeviceDetectionCloudEngine engine = + buildEngine("device", "hardwarevendor")) { + DeviceDataCloud data = process( + engine, + "{ 'device': { 'another': 'Example' }, " + + "'javascriptProperties': [] }"); + try { + data.getHardwareVendor(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + // The response's part did not hold the property. + } + } + } + + /** + * A null with no reason beside it is no value with the message Unknown. + */ + @Test + public void NullWithoutReason_MessageIsUnknown() throws Exception { + try (DeviceDetectionCloudEngine engine = + buildEngine("device", "hardwarevendor")) { + DeviceDataCloud data = process( + engine, + "{ 'device': { 'hardwarevendor': null }, " + + "'javascriptProperties': [] }"); assertFalse(data.getHardwareVendor().hasValue()); + assertEquals("Unknown", + data.getHardwareVendor().getNoValueMessage()); } } diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/WritesAfterProcessingTests.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/WritesAfterProcessingTests.java new file mode 100644 index 000000000..49f090c0f --- /dev/null +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/WritesAfterProcessingTests.java @@ -0,0 +1,186 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.cloud.flowelements; + +import fiftyone.devicedetection.cloud.data.DeviceDataCloud; +import fiftyone.devicedetection.cloud.data.MultiDeviceDataCloud; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudAspectEngineBase; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.TryGetResult; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.FlowElement.DataFactory; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.engines.data.AspectDataBase; +import org.junit.Test; +import org.junit.function.ThrowingRunnable; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The check, for the Flow Elements in this module, that processes each one, + * then writes to its Element Data and expects the error: the + * {@link DeviceDetectionCloudEngine} and the + * {@link HardwareProfileCloudEngine}, whose profiles are written only while + * it processes as well. Each is processed through its public process method + * with a fixed cloud response, so the check needs no resource key. + * @see Specification + */ +public class WritesAfterProcessingTests { + + private static final ILoggerFactory LOGGER_FACTORY = + LoggerFactory.getILoggerFactory(); + + private static void assertFails(String what, ThrowingRunnable write) { + assertThrows(what, IllegalStateException.class, write); + } + + /** + * Every write method of Element Data, and of Aspect Data, fails once its + * Flow Element has finished. + * @param data the Element Data of a finished Flow Element + */ + static void assertWritesFail(ElementData data) { + assertFails("put", () -> data.put("after", "value")); + assertFails("populateFromMap", () -> data.populateFromMap( + Collections.singletonMap("after", "value"))); + if (data instanceof AspectDataBase) { + AspectDataBase aspect = (AspectDataBase) data; + assertFails("setCloudRequestFailed", aspect::setCloudRequestFailed); + assertFails("addEngine", () -> aspect.addEngine(null)); + assertFails("addProcessCallable", + () -> aspect.addProcessCallable(null)); + } + } + + /** + * Attach the engine to a Pipeline whose Cloud Request Engine reports the + * product given. + */ + private static > T attach( + T engine, + String productKey, + PropertyMetaData property) { + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Collections.singletonList(property); + Map products = new HashMap<>(); + products.put(productKey, product); + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(products); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + engine.addPipeline(pipeline); + return engine; + } + + /** + * Process the engine through its public process method, with a Flow + * Data stand-in that holds the Element Data the engine adds and gives + * the Cloud Request Engine's data with the response given. + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + private static T process( + FlowElement engine, + String response) throws Exception { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getProcessStarted()).thenReturn(true); + when(cloudData.getJsonResponse()).thenReturn(response); + final FlowData flowData = mock(FlowData.class); + final List added = new ArrayList<>(); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + when(flowData.getOrAdd(any(TypedKey.class), any(DataFactory.class))) + .thenAnswer(invocation -> { + if (added.isEmpty()) { + DataFactory factory = invocation.getArgument(1); + added.add(factory.create(flowData)); + } + return added.get(0); + }); + when(flowData.tryGetValue(any(TypedKey.class))) + .thenAnswer(invocation -> { + TryGetResult result = new TryGetResult<>(); + if (added.isEmpty() == false) { + result.setValue(added.get(0)); + } + return result; + }); + engine.process(flowData); + assertEquals(1, added.size()); + return (T) added.get(0); + } + + @Test + public void deviceDetectionCloudEngine_ElementDataRefusesWrites() + throws Exception { + try (DeviceDetectionCloudEngine engine = attach( + new DeviceDetectionCloudEngineBuilder(LOGGER_FACTORY).build(), + "device", + new PropertyMetaData("hardwarevendor", "String", "cat", null))) { + DeviceDataCloud own = process( + engine, + "{ \"device\": { \"hardwarevendor\": \"Example\" } }"); + assertEquals("Example", own.getHardwareVendor().getValue()); + assertWritesFail(own); + } + } + + @Test + public void hardwareProfileCloudEngine_ElementDataRefusesWrites() + throws Exception { + try (HardwareProfileCloudEngine engine = attach( + new HardwareProfileCloudEngineBuilder(LOGGER_FACTORY).build(), + "hardware", + new PropertyMetaData("profiles", "Array", "cat", + Collections.singletonList(new PropertyMetaData( + "hardwarevendor", "String", "cat", null))))) { + MultiDeviceDataCloud own = process( + engine, + "{ \"hardware\": { \"profiles\": [ " + + "{ \"hardwarevendor\": \"Example\" } ] } }"); + assertEquals(1, own.getProfiles().size()); + assertWritesFail(own); + assertFails("addProfile", () -> own.addProfile(null)); + assertWritesFail(own.getProfiles().get(0)); + } + } +} diff --git a/device-detection/device-detection.hash.engine.on-premise/pom.xml b/device-detection/device-detection.hash.engine.on-premise/pom.xml index 7d141d302..b855aab87 100644 --- a/device-detection/device-detection.hash.engine.on-premise/pom.xml +++ b/device-detection/device-detection.hash.engine.on-premise/pom.xml @@ -11,7 +11,7 @@ device-detection.hash.engine.on-premise 51Degrees :: Device Detection :: Hash :: On Premise - High-performance on-premise hash-based device detection engine for Java, built for the 51Degrees Pipeline API. Parses HTTP headers locally to identify device hardware, operating systems, browsers, and crawlers, with no external service calls. + On-premise Device Detection engine and User-Agent parser for Java. Reads a 51Degrees Hash data file on your server to detect the device type and model, browser, operating system, and crawlers from the User-Agent and User-Agent Client Hints. Built on the 51Degrees Pipeline API. An alternative to UAParser.js, DeviceAtlas, and WURFL. https://github.com/51Degrees/51degrees-java diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/Constants.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/Constants.java index 8c4086162..39734c8cd 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/Constants.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/Constants.java @@ -26,7 +26,10 @@ public class Constants { public static class MatchMetrics { /* Category */ public static final String CATEGORY = "Device Metrics"; - + + /* Component */ + public static final String COMPONENT = "Metrics"; + /* Names */ public static final String MATCHED_NODES = "MatchedNodes"; public static final String DIFFERENCE = "Difference"; diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java index 691761125..bd44a63fc 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java @@ -44,6 +44,12 @@ /** * Internal implementation of the {@link DeviceDataHash} interface. This can * only be constructed by the {@link DeviceDetectionHashEngine}. + *

+ * Values are read from the native results with no lock. Reading a value + * writes into the results, so an instance is not safe to use from several + * threads at once, and a caller that shares one is responsible for any + * locking. + * @see Results and threads * @see Specification */ public class DeviceDataHashDefault @@ -259,6 +265,14 @@ private AspectPropertyValue> getUserAgentsInternal() { @Override protected boolean propertyIsAvailable(String propertyName) { checkState(); + // A property the engine was created with has a position, and the + // read goes by it, so there is no need to look the name up in the + // results first. A property the data set lacks keeps its position + // and reads as having no value. With results from more than one + // engine, the name finds the results that hold the property. + if (resultsList.size() == 1 && indexOf(propertyName) != null) { + return true; + } for (ResultsHashSwig results : resultsList) { if (results.containsProperty(Swig.asBytes(propertyName))) { return true; diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java index 0b5c93fb1..e9b256ae1 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java @@ -53,7 +53,14 @@ public class DeviceDetectionHashEngine extends FiftyOneOnPremiseAspectEngineBase { - private EngineHashSwig engine = null; + // Volatile, so a thread that processes sees the native engine a + // refresh on another thread made. + private volatile EngineHashSwig engine = null; + // Set, while this engine's monitor is held, once closing has freed the + // native objects, so a refresh that gets the monitor afterwards fails + // instead of using them. It has no initializer, so an engine built by + // a mock that skips field initializers still has it, as false. + private boolean freed; private final List properties = new ArrayList<>(); private final ConfigHashSwig config; private final RequiredPropertiesConfigSwig propertiesConfigSwig; @@ -83,6 +90,11 @@ public class DeviceDetectionHashEngine */ private volatile boolean propertyIndexesFixed; + /** + * The Metrics component, made the first time it is needed. + */ + private volatile MetricsComponentMetaData metricsComponent; + /** * Construct a new instance of the {@link DeviceDetectionHashEngine}. * @param logger logger instance to use for logging @@ -259,11 +271,42 @@ public ProfileMetaData getProfile(int profileId) { null : new ProfileMetaDataHash(this, profile); } + /** + * Get the ids of the profiles that hold a value of a property, such as a + * TAC or a native model name, from the native engine. The property name + * and the value are both matched exactly, as the data file holds them, + * so a value in another case does not match. The ids are in ascending + * order and each appears once. A property named with + * {@link DeviceDetectionHashEngineBuilder#setIndexedProperties(List)} is + * found through the native index, and any other by reading every + * profile, which gives the same ids more slowly. + * @see Specification + * @param propertyName name of the property, such as TAC or NativeModel + * @param valueName the value of the property + * @return ids of the profiles holding the value, empty when none do + */ + public List getProfileIdsForPropertyValue( + String propertyName, + String valueName) { + List result = new ArrayList<>(); + try (VectorUInt32Swig ids = + engine.getProfileIdsForPropertyValue(propertyName, valueName)) { + int size = ids.size(); + for (int i = 0; i < size; i++) { + result.add(ids.get(i)); + } + } + return result; + } + @Override public CloseableIterable getComponents() { + // The Metrics component holds the match metrics, which the data file + // does not have, so it is given after the data file's components. return new ComponentIterable( this, - engine.getMetaData().getComponents()); + engine.getMetaData().getComponents(), + getMetricsComponent()); } @Override @@ -334,8 +377,42 @@ public EvidenceKeyFilter getEvidenceKeyFilter() { return evidenceKeyFilter; } + /** + * {@inheritDoc} + * Only one refresh of this engine runs at a time, from either overload, + * whichever route calls it. A second refresh waits for the first to + * finish. Processing takes no part in this and never waits for a + * refresh. + *

+ * The lock is this engine's own monitor, held for the whole refresh. + * Closing the engine takes it too, so the native objects are freed + * only once a refresh that is running has finished. Nothing else in + * the engine or the Pipeline takes it, and no other lock is taken + * while it is held, so it cannot deadlock. It is not a + * field, so an engine built by a mock that skips field initializers + * still has it. + */ @Override public void refreshData(String dataFileIdentifier) { + synchronized (this) { + refuseIfFreed(); + refreshFromFile(); + } + } + + /** + * Called with this engine's monitor held, before a refresh uses the + * native objects. + * @throws IllegalStateException if the engine has been closed + */ + private void refuseIfFreed() { + if (freed) { + throw new IllegalStateException( + "The engine has been closed, so its data cannot be refreshed."); + } + } + + private void refreshFromFile() { AspectEngineDataFile dataFile = getDataFiles().get(0); if (Objects.isNull(engine)) { @@ -367,14 +444,22 @@ public void refreshData(String dataFileIdentifier) { setEngineMetaData(); } + /** + * {@inheritDoc} + * Only one refresh of this engine runs at a time, as + * {@link #refreshData(String)} describes. + */ @Override public void refreshData(String dataFileIdentifier, byte[] data) { - if (engine == null) { - engine = new EngineHashSwig(data, config, propertiesConfigSwig); - } else { - engine.refreshData(data); + synchronized (this) { + refuseIfFreed(); + if (engine == null) { + engine = new EngineHashSwig(data, config, propertiesConfigSwig); + } else { + engine.refreshData(data); + } + setEngineMetaData(); } - setEngineMetaData(); } @Override @@ -384,17 +469,19 @@ protected void processEngine(FlowData flowData, DeviceDataHash deviceData) { List keys = evidenceKeys; for (Map.Entry evidenceItem : flowData.getEvidence().asKeyMap().entrySet()) { - boolean containsKey = false; + String matchedKey = null; for (String key : keys) { if (key.equalsIgnoreCase(evidenceItem.getKey())) { - containsKey = true; + matchedKey = key; break; } } - if (containsKey) { + if (matchedKey != null) { + // The native code matches the prefix with case, so it + // is given the key as the native code spells it. relevantEvidence .addFromBytes( - Swig.asBytes(evidenceItem.getKey()), + Swig.asBytes(matchedKey), Swig.asBytes(evidenceItem.getValue().toString())); } } @@ -412,14 +499,23 @@ protected void unmanagedResourcesCleanup() { } } } - if (config != null) { - config.delete(); - } - if (propertiesConfigSwig != null) { - propertiesConfigSwig.delete(); - } - if (engine != null) { - engine.delete(); + // A refresh holds this engine's monitor while it uses the native + // objects, so taking it here waits for a refresh that is running + // before they are freed. + synchronized (this) { + if (freed) { + return; + } + freed = true; + if (config != null) { + config.delete(); + } + if (propertiesConfigSwig != null) { + propertiesConfigSwig.delete(); + } + if (engine != null) { + engine.delete(); + } } } @@ -447,6 +543,32 @@ private void setEngineMetaData() { * @return meta data for metric properties */ private List getMetricProperties() { + return getMetricsComponent().getPropertyList(); + } + + /** + * Get the Metrics component, which holds the match metric properties. It + * does not depend on the data file, so it is made once. Two threads that + * both find it missing each make one the same as the other. + * @return the Metrics component + */ + private MetricsComponentMetaData getMetricsComponent() { + MetricsComponentMetaData component = metricsComponent; + if (component == null) { + component = newMetricsComponent(); + metricsComponent = component; + } + return component; + } + + /** + * Make the Metrics component and the match metric properties in it, which + * the native code does not include in its metadata, as the table of match + * metric properties gives them. + * @return new Metrics component + */ + private MetricsComponentMetaData newMetricsComponent() { + MetricsComponentMetaData component = new MetricsComponentMetaData(); List dataFileList = Arrays.asList( "Lite", "Premium", "Enterprise", "TAC"); FiftyOneAspectPropertyMetaData[] metricProperties = @@ -467,7 +589,7 @@ private List getMetricProperties() { false, false, Constants.MatchMetrics.MATCHED_NODES_DESCRIPTION, - null, + component, Arrays.asList(), new ValueMetaDataDefault("0")), new FiftyOneAspectPropertyMetaDataDefault( @@ -485,7 +607,7 @@ private List getMetricProperties() { false, false, Constants.MatchMetrics.DIFFERENCE_DESCRIPTION, - null, + component, Arrays.asList(), new ValueMetaDataDefault("0")), new FiftyOneAspectPropertyMetaDataDefault( @@ -503,7 +625,7 @@ private List getMetricProperties() { false, false, Constants.MatchMetrics.DRIFT_DESCRIPTION, - null, + component, Arrays.asList(), new ValueMetaDataDefault("0")), new FiftyOneAspectPropertyMetaDataDefault( @@ -521,7 +643,7 @@ private List getMetricProperties() { false, false, Constants.MatchMetrics.DEVICE_ID_DESCRIPTION, - null, + component, Arrays.asList(), new ValueMetaDataDefault("0-0-0-0")), new FiftyOneAspectPropertyMetaDataDefault( @@ -539,7 +661,7 @@ private List getMetricProperties() { false, false, Constants.MatchMetrics.USER_AGENTS_DESCRIPTION, - null, + component, Arrays.asList(), new ValueMetaDataDefault("n/a")), new FiftyOneAspectPropertyMetaDataDefault( @@ -557,7 +679,7 @@ private List getMetricProperties() { false, false, Constants.MatchMetrics.ITERATIONS_DESCRIPTION, - null, + component, Arrays.asList(), new ValueMetaDataDefault("0")), new FiftyOneAspectPropertyMetaDataDefault( @@ -570,12 +692,12 @@ private List getMetricProperties() { null, (byte) 0, true, - true, + false, false, false, false, Constants.MatchMetrics.METHOD_DESCRIPTION, - null, + component, Arrays.asList( new ValueMetaDataDefault("NONE"), new ValueMetaDataDefault("PERFORMANCE"), @@ -583,7 +705,10 @@ private List getMetricProperties() { new ValueMetaDataDefault( "PREDICTIVE")), new ValueMetaDataDefault("NONE")) }; - return Arrays.asList(metricProperties); + for (FiftyOneAspectPropertyMetaData property : metricProperties) { + component.addProperty(property); + } + return component; } @Override diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngineBuilder.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngineBuilder.java index 1de52bba8..0e6f98760 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngineBuilder.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngineBuilder.java @@ -156,6 +156,7 @@ public DeviceDetectionHashEngineBuilder setPerformanceProfile( } } + @DefaultValue("Balanced") @Override public DeviceDetectionHashEngineBuilder setPerformanceProfile( PerformanceProfiles profile) { @@ -285,6 +286,49 @@ public DeviceDetectionHashEngineBuilder setUsePredictiveGraph(boolean use) { return this; } + /** + * Set the properties, such as TAC and NativeModel, whose values are + * indexed to the profiles holding them when the data file loads, so that + * {@link DeviceDetectionHashEngine#getProfileIdsForPropertyValue(String, String)} + * finds the profiles for a value of one of them without reading every + * profile. Names are matched exactly. A refresh indexes the same + * properties. + *

+ * Default is none. + * @see Specification + * @param names of the properties to index + * @return this builder + */ + @DefaultValue("None") + public DeviceDetectionHashEngineBuilder setIndexedProperties( + List names) { + try (VectorStringSwig namesSwig = new VectorStringSwig()) { + namesSwig.addAll(names); + config.setIndexedProperties(namesSwig); + } + return this; + } + + /** + * Set the properties whose values are indexed to the profiles holding + * them, as {@link #setIndexedProperties(List)} does, from a comma + * separated list of names, as a configuration file gives them. + * @param names comma separated names of the properties to index + * @return this builder + */ + @DefaultValue("None") + public DeviceDetectionHashEngineBuilder setIndexedProperties( + String names) { + List list = new ArrayList<>(); + for (String name : names.split(",")) { + if (name.trim().isEmpty() == false) { + list.add(name.trim()); + } + } + return setIndexedProperties(list); + } + + @DefaultValue("No results cache, and none can be set for this engine") @Override public DeviceDetectionHashEngineBuilder setCache( CacheConfiguration cacheConfiguration) { diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/MetricsComponentMetaData.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/MetricsComponentMetaData.java new file mode 100644 index 000000000..f4572c409 --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/MetricsComponentMetaData.java @@ -0,0 +1,115 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterableDefault; +import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; +import fiftyone.pipeline.engines.fiftyone.data.ProfileMetaData; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * The Metrics component, which holds the match metric properties. The data + * file has no such component, so the engine adds it to the components the + * native code gives, and each match metric property names it as its + * component. + * @see Specification + */ +class MetricsComponentMetaData implements ComponentMetaData { + + /** + * Id of the Metrics component. The data file's components are numbered + * from 1, so this is the largest id a component can have, which no data + * file component uses. + */ + static final byte COMPONENT_ID = (byte) 255; + + private final List properties = + new ArrayList<>(); + + /** + * Add a match metric property to the component. Called only while the + * engine makes its match metric properties. + * @param property to add + */ + void addProperty(FiftyOneAspectPropertyMetaData property) { + properties.add(property); + } + + /** + * The match metric properties, in the order they were added. Holds no + * native memory, so there is nothing to close. + * @return unmodifiable list of the properties + */ + List getPropertyList() { + return Collections.unmodifiableList(properties); + } + + @Override + public byte getComponentId() { + return COMPONENT_ID; + } + + @Override + public String getName() { + return Constants.MatchMetrics.COMPONENT; + } + + /** + * The match metrics have no profiles. + * @return null + */ + @Override + public ProfileMetaData getDefaultProfile() { + return null; + } + + @Override + public CloseableIterable getProperties() { + return new CloseableIterableDefault<>( + Collections.unmodifiableList(properties)); + } + + @Override + public FiftyOneAspectPropertyMetaData getProperty(String propertyName) { + for (FiftyOneAspectPropertyMetaData property : properties) { + if (property.getName().equalsIgnoreCase(propertyName)) { + return property; + } + } + return null; + } + + @Override + public void close() { + } + + @Override + public String toString() { + return getName(); + } +} diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/interop/ComponentIterable.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/interop/ComponentIterable.java index bee78cd10..77d748d2a 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/interop/ComponentIterable.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/interop/ComponentIterable.java @@ -39,6 +39,10 @@ public class ComponentIterable private final DeviceDetectionHashEngine engine; + private final long nativeSize; + + private final ComponentMetaData[] added; + /** * Create a new instance. * @param engine the engine which the meta data relates to @@ -47,14 +51,41 @@ public class ComponentIterable public ComponentIterable( DeviceDetectionHashEngine engine, ComponentMetaDataCollectionSwig collection) { - super(collection.getSize()); + this(engine, collection, collection.getSize()); + } + + /** + * Create a new instance which gives the native components and then the + * components given, such as the Metrics component, which the data file + * does not have. + * @param engine the engine which the meta data relates to + * @param collection the native collection of components + * @param added components to give after the native ones + */ + public ComponentIterable( + DeviceDetectionHashEngine engine, + ComponentMetaDataCollectionSwig collection, + ComponentMetaData... added) { + this(engine, collection, collection.getSize(), added); + } + + private ComponentIterable( + DeviceDetectionHashEngine engine, + ComponentMetaDataCollectionSwig collection, + long nativeSize, + ComponentMetaData... added) { + super(nativeSize + added.length); this.engine = engine; this.collection = collection; + this.nativeSize = nativeSize; + this.added = added; } @Override protected ComponentMetaData get(long index) { - return new ComponentMetaDataHash(engine, collection.getByIndex(index)); + return index < nativeSize ? + new ComponentMetaDataHash(engine, collection.getByIndex(index)) : + added[(int) (index - nativeSize)]; } @Override diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/TestsBase.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/TestsBase.java index f3826ee0e..25a36a709 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/TestsBase.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/TestsBase.java @@ -23,10 +23,10 @@ package fiftyone.devicedetection.hash.engine.onpremise; import fiftyone.devicedetection.shared.testhelpers.UserAgentGenerator; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; +import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; import fiftyone.pipeline.engines.Constants; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_PATH; +import static fiftyone.devicedetection.shared.testhelpers.TestFileUtils.UA_FILE_PATH; import static fiftyone.pipeline.util.FileFinder.getFilePath; public class TestsBase { @@ -44,7 +44,7 @@ protected UserAgentGenerator getUserAgents() { protected void testInitialize(Constants.PerformanceProfiles profile) throws Exception { wrapper = new WrapperHash( - FileUtils.getHashFile(), + TestFileUtils.getHashFile(), profile); userAgents = new UserAgentGenerator( getFilePath(UA_FILE_PATH)); diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHasherHash.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHasherHash.java index aa5a619ba..07c97024d 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHasherHash.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/data/MetaDataHasherHash.java @@ -22,7 +22,6 @@ package fiftyone.devicedetection.hash.engine.onpremise.data; -import fiftyone.devicedetection.hash.engine.onpremise.flowelements.Constants; import fiftyone.devicedetection.shared.testhelpers.Wrapper; import fiftyone.devicedetection.shared.testhelpers.data.MetaDataHasher; import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; @@ -51,10 +50,7 @@ public int hashProperties(int hash, Wrapper wrapper) throws Exception { j++; } } - // Match metric property does not have component so don't include - if (!property.getCategory().equals(Constants.MatchMetrics.CATEGORY)) { - hash ^= property.getComponent().hashCode(); - } + hash ^= property.getComponent().hashCode(); if (property.getDefaultValue() != null) { hash ^= property.getDefaultValue().hashCode(); } @@ -93,7 +89,10 @@ public int hashComponents(int hash, Wrapper wrapper) throws Exception { i++; } } - hash ^= component.getDefaultProfile().hashCode(); + // The Metrics component has no profiles, so no default. + if (component.getDefaultProfile() != null) { + hash ^= component.getDefaultProfile().hashCode(); + } } } return hash; diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DataRefreshTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DataRefreshTests.java new file mode 100644 index 000000000..9e61d79af --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DataRefreshTests.java @@ -0,0 +1,311 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.devicedetection.hash.engine.onpremise.TestsBase; +import fiftyone.devicedetection.shared.DeviceData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Refreshing the engine's data, with the free Lite data file. Refreshes + * from several threads at once leave a working engine, a refresh from data + * that cannot be used fails and leaves the data the engine had, and + * processing while the data is refreshed gives the same values. + */ +public class DataRefreshTests extends TestsBase { + + private static final int THREADS = 4; + + private static final int REFRESHES = 5; + + private static final long WAIT_SECONDS = 300; + + private static final List USER_AGENTS = Arrays.asList( + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) " + + "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 " + + "Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); + + private ExecutorService executor; + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.MaxPerformance); + executor = Executors.newFixedThreadPool(THREADS); + } + + @After + public void cleanup() { + executor.shutdownNow(); + testCleanup(); + } + + /** + * Several threads refresh the engine from its data file at once, and + * the engine then gives the values it gave before. + */ + @Test + public void Refresh_FromSeveralThreads_EngineStillWorks() + throws Exception { + List expected = isMobile(); + DeviceDetectionHashEngine engine = getWrapper().getEngine(); + String identifier = engine.getDataFileMetaData().getIdentifier(); + CountDownLatch start = new CountDownLatch(1); + List> refreshers = new ArrayList<>(); + for (int t = 0; t < THREADS; t++) { + refreshers.add(executor.submit(() -> { + start.await(); + for (int i = 0; i < REFRESHES; i++) { + engine.refreshData(identifier); + } + return REFRESHES; + })); + } + start.countDown(); + for (Future refresher : refreshers) { + assertEquals(Integer.valueOf(REFRESHES), + refresher.get(WAIT_SECONDS, TimeUnit.SECONDS)); + } + assertEquals(expected, isMobile()); + } + + /** + * Several threads refresh the engine at once from the same data held in + * memory, and every refresh completes and the engine then gives the + * values it gave before. A refresh from memory leaves the native engine + * without a file path, so this test doesn't mix it with refreshes from + * the file. + */ + @Test + public void Refresh_FromMemoryOnSeveralThreads_EngineStillWorks() + throws Exception { + List expected = isMobile(); + DeviceDetectionHashEngine engine = getWrapper().getEngine(); + String identifier = engine.getDataFileMetaData().getIdentifier(); + byte[] data = Files.readAllBytes(Paths.get( + engine.getDataFileMetaData().getDataFilePath())); + CountDownLatch start = new CountDownLatch(1); + List> refreshers = new ArrayList<>(); + for (int t = 0; t < THREADS; t++) { + refreshers.add(executor.submit(() -> { + start.await(); + for (int i = 0; i < REFRESHES; i++) { + engine.refreshData(identifier, data); + } + return REFRESHES; + })); + } + start.countDown(); + for (Future refresher : refreshers) { + assertEquals(Integer.valueOf(REFRESHES), + refresher.get(WAIT_SECONDS, TimeUnit.SECONDS)); + } + assertEquals(expected, isMobile()); + } + + /** + * A refresh from data that is not a data file fails with an error, and + * the engine goes on using the data it had. + */ + @Test + public void Refresh_DataNotUsable_FailsAndKeepsTheData() throws Exception { + List expected = isMobile(); + DeviceDetectionHashEngine engine = getWrapper().getEngine(); + String published = String.valueOf( + engine.getDataFileMetaData().getDataPublishedDateTime()); + byte[] notData = "This is not a data file.".getBytes( + StandardCharsets.UTF_8); + + boolean failed = false; + try { + engine.refreshData( + engine.getDataFileMetaData().getIdentifier(), + notData); + } + catch (RuntimeException e) { + failed = true; + } + + assertTrue("A refresh from data that cannot be used did not fail.", + failed); + assertEquals(expected, isMobile()); + assertEquals(published, String.valueOf( + engine.getDataFileMetaData().getDataPublishedDateTime())); + } + + /** + * Threads process while another refreshes the engine from its data + * file, and every process gives the values the engine gave before. + */ + @Test + public void Process_WhileRefreshing_GivesTheSameValues() + throws Exception { + List expected = isMobile(); + DeviceDetectionHashEngine engine = getWrapper().getEngine(); + String identifier = engine.getDataFileMetaData().getIdentifier(); + AtomicBoolean refreshing = new AtomicBoolean(true); + CountDownLatch start = new CountDownLatch(1); + List> processors = new ArrayList<>(); + for (int t = 0; t < THREADS - 1; t++) { + processors.add(executor.submit(process(expected, refreshing, start))); + } + Future refresher = executor.submit(() -> { + start.await(); + try { + for (int i = 0; i < REFRESHES; i++) { + engine.refreshData(identifier); + } + } + finally { + refreshing.set(false); + } + return REFRESHES; + }); + start.countDown(); + assertEquals(Integer.valueOf(REFRESHES), + refresher.get(WAIT_SECONDS, TimeUnit.SECONDS)); + for (Future processor : processors) { + assertTrue("A thread processed nothing during the refreshes.", + processor.get(WAIT_SECONDS, TimeUnit.SECONDS) > 0); + } + } + + private Callable process( + List expected, + AtomicBoolean refreshing, + CountDownLatch start) { + return () -> { + start.await(); + int processed = 0; + do { + List values = isMobile(); + if (expected.equals(values) == false) { + fail("Processing during a refresh gave " + values + + " where " + expected + " was given before."); + } + processed++; + } while (refreshing.get()); + return processed; + }; + } + + /** + * Whether each of the User-Agents is a mobile device. + */ + private List isMobile() throws Exception { + List values = new ArrayList<>(); + for (String userAgent : USER_AGENTS) { + try (FlowData data = + getWrapper().getPipeline().createFlowData()) { + data.addEvidence("header.user-agent", userAgent); + data.process(); + values.add( + data.get(DeviceData.class).getIsMobile().getValue()); + } + } + return values; + } + + /** + * Closing the engine while a refresh runs waits for the refresh to + * finish, so the native engine is never freed under it. The test holds + * the engine's own monitor, which a refresh holds while it runs. + */ + @Test + public void Close_WhileARefreshRuns_WaitsForTheRefresh() + throws Exception { + DeviceDetectionHashEngine engine = getWrapper().getEngine(); + CountDownLatch held = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + Future refresh = executor.submit(() -> { + synchronized (engine) { + held.countDown(); + return release.await(WAIT_SECONDS, TimeUnit.SECONDS); + } + }); + assertTrue(held.await(WAIT_SECONDS, TimeUnit.SECONDS)); + Future closing = executor.submit(() -> { + engine.close(); + return true; + }); + try { + closing.get(2, TimeUnit.SECONDS); + fail("The engine closed while a refresh was running."); + } catch (TimeoutException e) { + // Closing waits for the refresh, as it should. + } finally { + release.countDown(); + } + assertTrue(refresh.get(WAIT_SECONDS, TimeUnit.SECONDS)); + assertTrue(closing.get(WAIT_SECONDS, TimeUnit.SECONDS)); + } + + /** + * A refresh that gets the engine's monitor after the engine is closed, + * from either overload, fails with an error and does not use the native + * objects that closing freed. + */ + @Test + public void Refresh_AfterClose_FailsWithoutUsingTheFreedEngine() + throws Exception { + DeviceDetectionHashEngine engine = getWrapper().getEngine(); + String identifier = engine.getDataFileMetaData().getIdentifier(); + engine.close(); + try { + engine.refreshData(identifier); + fail("A refresh from file ran after the engine was closed."); + } catch (IllegalStateException e) { + // As it should. + } + try { + engine.refreshData(identifier, new byte[] { 0 }); + fail("A refresh from memory ran after the engine was closed."); + } catch (IllegalStateException e) { + // As it should. + } + } +} diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineOverMockNativeCode.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineOverMockNativeCode.java new file mode 100644 index 000000000..c66d8421b --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineOverMockNativeCode.java @@ -0,0 +1,129 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.ComponentMetaDataCollectionSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.EngineHashSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.MetaDataSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.PropertyMetaDataCollectionSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.VectorStringSwig; +import org.slf4j.Logger; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Map; +import java.util.TreeMap; + +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +/** + * Makes a {@link DeviceDetectionHashEngine} whose native engine is a mock + * with no properties and no components of its own, so the parts of the + * engine written in Java can be tested without the native library or a data + * file. The engine's own methods run as written. Its constructor does not + * run, as it loads a data file, so the fields it would set are set here. + */ +class EngineOverMockNativeCode { + + final DeviceDetectionHashEngine engine; + + final EngineHashSwig nativeEngine; + + EngineOverMockNativeCode() throws Exception { + nativeEngine = mock(EngineHashSwig.class); + MetaDataSwig metaData = mock(MetaDataSwig.class); + PropertyMetaDataCollectionSwig properties = + mock(PropertyMetaDataCollectionSwig.class); + when(properties.getSize()).thenReturn(0L); + ComponentMetaDataCollectionSwig components = + mock(ComponentMetaDataCollectionSwig.class); + when(components.getSize()).thenReturn(0L); + when(metaData.getProperties()).thenReturn(properties); + when(metaData.getComponents()).thenReturn(components); + when(nativeEngine.getMetaData()).thenReturn(metaData); + + engine = mock( + DeviceDetectionHashEngine.class, + withSettings().defaultAnswer(CALLS_REAL_METHODS)); + set("engine", nativeEngine); + set("properties", new ArrayList<>()); + set("logger", mock(Logger.class)); + // No data file, so a refresh from memory updates no file metadata. + set("dataFiles", new ArrayList<>()); + // The property indexes are fixed, as they are after the first load. + set("propertyIndexes", Collections.emptyMap()); + set("propertyIndexesFixed", true); + setKeys(); + } + + /** + * Set the property names the engine was created with, so each name's + * position is its place in the list. + * @param names the names + */ + void setPropertyNames(String... names) throws Exception { + Map indexes = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (int i = 0; i < names.length; i++) { + indexes.putIfAbsent(names[i], i); + } + set("propertyIndexes", indexes); + } + + /** + * Set the evidence keys the native engine gives from now on. + * @param keys the keys + */ + void setKeys(String... keys) { + VectorStringSwig vector = mock( + VectorStringSwig.class, + withSettings().defaultAnswer(CALLS_REAL_METHODS)); + doReturn(keys.length).when(vector).size(); + for (int i = 0; i < keys.length; i++) { + doReturn(keys[i]).when(vector).get(i); + } + doNothing().when(vector).delete(); + when(nativeEngine.getKeys()).thenReturn(vector); + } + + private void set(String name, Object value) throws Exception { + Class type = DeviceDetectionHashEngine.class; + while (type != null) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + field.set(engine, value); + return; + } catch (NoSuchFieldException e) { + type = type.getSuperclass(); + } + } + throw new NoSuchFieldException(name); + } +} diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java index 198e43bb2..c3c0ae5f8 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java @@ -34,11 +34,11 @@ import org.slf4j.ILoggerFactory; import org.slf4j.LoggerFactory; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; +import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; import fiftyone.pipeline.engines.Constants; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.TAC_HASH_DATA_FILE_NAME; +import static fiftyone.devicedetection.shared.testhelpers.TestFileUtils.LITE_HASH_DATA_FILE_PATH; +import static fiftyone.devicedetection.shared.testhelpers.TestFileUtils.TAC_HASH_DATA_FILE_NAME; import static fiftyone.pipeline.util.FileFinder.getFilePath; public class EngineTests { diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EvidenceKeysOnRefreshTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EvidenceKeysOnRefreshTests.java new file mode 100644 index 000000000..94559c186 --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EvidenceKeysOnRefreshTests.java @@ -0,0 +1,80 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * After loading the data source, call getKeys and store the accepted + * evidence keys. This MUST be done at start-up and any time the data is + * refreshed + * (device-detection-specification/pipeline-elements/device-detection-on-premise.md + * line 86 at 706de1f0). The native engine here is a mock whose keys change + * between refreshes, as they do when a refresh loads a data file with other + * keys. Start-up with a real data file is covered by EvidenceKeysTests. + */ +public class EvidenceKeysOnRefreshTests { + + @Test + public void EvidenceKeys_ReadAgainOnRefresh() throws Exception { + EngineOverMockNativeCode mocks = new EngineOverMockNativeCode(); + DeviceDetectionHashEngine engine = mocks.engine; + + mocks.setKeys("header.user-agent"); + engine.refreshData(null, new byte[0]); + EvidenceKeyFilter first = engine.getEvidenceKeyFilter(); + assertTrue(first.include("header.user-agent")); + // Not case sensitive. + assertTrue(first.include("header.User-Agent")); + assertFalse(first.include("header.sec-ch-ua")); + + mocks.setKeys("header.user-agent", "header.sec-ch-ua"); + engine.refreshData(null, new byte[0]); + EvidenceKeyFilter second = engine.getEvidenceKeyFilter(); + assertTrue("A key of the new data was not accepted after refresh", + second.include("header.sec-ch-ua")); + verify(mocks.nativeEngine, times(2)).refreshData(new byte[0]); + } + + /** + * The keys are stored, so using the filter does not call the native code. + */ + @Test + public void EvidenceKeys_StoredNotReadForEachUse() throws Exception { + EngineOverMockNativeCode mocks = new EngineOverMockNativeCode(); + mocks.setKeys("header.user-agent"); + mocks.engine.refreshData(null, new byte[0]); + clearInvocations(mocks.nativeEngine); + for (int i = 0; i < 10; i++) { + mocks.engine.getEvidenceKeyFilter().include("header.user-agent"); + } + verify(mocks.nativeEngine, times(0)).getKeys(); + } +} diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/MatchMetricMetaDataTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/MatchMetricMetaDataTests.java new file mode 100644 index 000000000..e0b417302 --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/MatchMetricMetaDataTests.java @@ -0,0 +1,148 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; + +/** + * Metadata for the match metric Properties MUST be added, as they will not be + * included in the metadata exposed by the native code + * (device-detection-specification/pipeline-elements/device-detection-on-premise.md + * line 452 at 706de1f0). The table that follows gives each one's Category, + * Available With, Component, type and default value. The native engine here + * is a mock with no properties, so every property the engine lists is one it + * added itself. + */ +public class MatchMetricMetaDataTests { + + private static final List NAMES = Arrays.asList( + "MatchedNodes", "Difference", "Drift", "DeviceId", "UserAgents", + "Iterations", "Method"); + + private static List metrics() + throws Exception { + return new EngineOverMockNativeCode().engine.getProperties(); + } + + /** + * The engine lists the seven, in the order of the table. + */ + @Test + public void MatchMetrics_AllListedInTableOrder() throws Exception { + List metrics = metrics(); + assertEquals(NAMES.size(), metrics.size()); + for (int i = 0; i < NAMES.size(); i++) { + assertEquals(NAMES.get(i), metrics.get(i).getName()); + } + } + + /** + * Category = "Device Metrics" and Available With = "Lite", "Premium", + * "Enterprise", "TAC". + */ + @Test + public void MatchMetrics_CategoryAndAvailableWith() throws Exception { + for (FiftyOneAspectPropertyMetaData property : metrics()) { + assertEquals(property.getName(), + "Device Metrics", property.getCategory()); + assertEquals(property.getName(), + Arrays.asList("Lite", "Premium", "Enterprise", "TAC"), + property.getDataTiersWherePresent()); + } + } + + /** + * Component = "Metrics". + */ + @Test + public void MatchMetrics_ComponentIsMetrics() throws Exception { + for (FiftyOneAspectPropertyMetaData property : metrics()) { + assertNotNull( + property.getName() + " has no component", + property.getComponent()); + assertEquals(property.getName(), + "Metrics", property.getComponent().getName()); + } + } + + /** + * The types in the table: int, string, and list of strings for + * UserAgents only, so only UserAgents is a list. + */ + @Test + public void MatchMetrics_TypesAsTheTableGives() throws Exception { + for (FiftyOneAspectPropertyMetaData property : metrics()) { + switch (property.getName()) { + case "MatchedNodes": + case "Difference": + case "Drift": + case "Iterations": + assertEquals(property.getName(), + Integer.class, property.getType()); + assertFalse(property.getName(), property.getList()); + break; + case "DeviceId": + case "Method": + assertEquals(property.getName(), + String.class, property.getType()); + assertFalse(property.getName() + " is a string, " + + "not a list", property.getList()); + break; + case "UserAgents": + assertEquals(property.getName(), + List.class, property.getType()); + break; + default: + throw new AssertionError( + "Unexpected property " + property.getName()); + } + } + } + + /** + * The default values in the table, other than the list-valued UserAgents + * whose description in Property metadata is an open question. + */ + @Test + public void MatchMetrics_DefaultValues() throws Exception { + for (FiftyOneAspectPropertyMetaData property : metrics()) { + String expected; + switch (property.getName()) { + case "DeviceId": expected = "0-0-0-0"; break; + case "Method": expected = "NONE"; break; + case "UserAgents": continue; + default: expected = "0"; break; + } + assertEquals(property.getName(), + expected, property.getDefaultValue().getName()); + } + } +} diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/MetricsComponentTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/MetricsComponentTests.java new file mode 100644 index 000000000..233768981 --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/MetricsComponentTests.java @@ -0,0 +1,104 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.pipeline.engines.fiftyone.data.CloseableIterable; +import fiftyone.pipeline.engines.fiftyone.data.ComponentMetaData; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Component = "Metrics" - This component MUST also be added to the list of + * components returned by the Engine + * (device-detection-specification/pipeline-elements/device-detection-on-premise.md + * line 464 at 706de1f0). The native engine here is a mock with no components, + * so every component the engine lists is one it added itself. + */ +public class MetricsComponentTests { + + private static List components( + DeviceDetectionHashEngine engine) throws Exception { + List result = new ArrayList<>(); + try (CloseableIterable components = + engine.getComponents()) { + for (ComponentMetaData component : components) { + result.add(component); + } + } + return result; + } + + /** + * The engine's components include Metrics. + */ + @Test + public void Components_IncludeMetrics() throws Exception { + List components = + components(new EngineOverMockNativeCode().engine); + assertEquals(1, components.size()); + assertEquals("Metrics", components.get(0).getName()); + } + + /** + * The engine finds Metrics by its name. + */ + @Test + public void Components_GetByName() throws Exception { + ComponentMetaData metrics = + new EngineOverMockNativeCode().engine.getComponent("Metrics"); + assertNotNull("No component named Metrics", metrics); + assertEquals("Metrics", metrics.getName()); + } + + /** + * The Metrics component holds the match metric properties, each of which + * names Metrics as its component. + */ + @Test + public void Components_MetricsHoldsTheMatchMetrics() throws Exception { + DeviceDetectionHashEngine engine = + new EngineOverMockNativeCode().engine; + ComponentMetaData metrics = components(engine).get(0); + List names = new ArrayList<>(); + try (CloseableIterable properties = + metrics.getProperties()) { + for (FiftyOneAspectPropertyMetaData property : properties) { + names.add(property.getName()); + assertEquals(property.getName(), + "Metrics", property.getComponent().getName()); + } + } + assertEquals( + Arrays.asList("MatchedNodes", "Difference", "Drift", "DeviceId", + "UserAgents", "Iterations", "Method"), + names); + assertNotNull(metrics.getProperty("DeviceId")); + } +} diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/NativeCallsOncePerRefreshTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/NativeCallsOncePerRefreshTests.java new file mode 100644 index 000000000..10655e774 --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/NativeCallsOncePerRefreshTests.java @@ -0,0 +1,80 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import org.junit.Test; + +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * The main performance bottleneck is usually marshalling data to and from + * native code. As such, steps SHOULD be taken to minimize this as much as + * possible + * (device-detection-specification/pipeline-elements/device-detection-on-premise.md + * line 339 at 706de1f0), the last being that values which change only on a + * data refresh are stored in the engine until a refresh occurs. Counts the + * calls to a mock native engine. + */ +public class NativeCallsOncePerRefreshTests { + + /** + * Property metadata is read from the native code once, and again only + * after a refresh. + */ + @Test + public void PropertyMetaData_ReadOncePerRefresh() throws Exception { + EngineOverMockNativeCode mocks = new EngineOverMockNativeCode(); + mocks.engine.refreshData(null, new byte[0]); + clearInvocations(mocks.nativeEngine); + + for (int i = 0; i < 5; i++) { + mocks.engine.getProperties(); + } + verify(mocks.nativeEngine, times(1)).getMetaData(); + + mocks.engine.refreshData(null, new byte[0]); + for (int i = 0; i < 5; i++) { + mocks.engine.getProperties(); + } + verify(mocks.nativeEngine, times(2)).getMetaData(); + } + + /** + * The evidence keys and their filter are read once for each refresh. + */ + @Test + public void EvidenceKeys_ReadOncePerRefresh() throws Exception { + EngineOverMockNativeCode mocks = new EngineOverMockNativeCode(); + mocks.setKeys("header.user-agent"); + for (int refresh = 1; refresh <= 3; refresh++) { + mocks.engine.refreshData(null, new byte[0]); + for (int i = 0; i < 5; i++) { + mocks.engine.getEvidenceKeyFilter() + .include("header.user-agent"); + } + verify(mocks.nativeEngine, times(refresh)).getKeys(); + } + } +} diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/NoNamesTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/NoNamesTests.java new file mode 100644 index 000000000..5136272e1 --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/NoNamesTests.java @@ -0,0 +1,106 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.RequiredPropertiesConfigSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.VectorStringSwig; +import fiftyone.pipeline.engines.Constants; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static fiftyone.devicedetection.shared.testhelpers.TestFileUtils.LITE_HASH_DATA_FILE_PATH; +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +/** + * An implementation configured with no names MUST create the Engine with no + * names, and MUST NOT pass the data set's names itself + * (device-detection-specification/pipeline-elements/device-detection-on-premise.md + * line 189 at 706de1f0). Reads the names the native configuration the + * engine was created with holds, which is what decides whether the engine + * walks graphs during process, rather than the indexes the engine has + * afterwards, which are the same either way. + */ +public class NoNamesTests { + + private static final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + private static File liteFile() { + try { + return getFilePath(LITE_HASH_DATA_FILE_PATH); + } + catch (IllegalArgumentException e) { + assumeTrue(e.getMessage(), false); + return null; + } + } + + private static List namesCreatedWith( + File file, List names) throws Exception { + DeviceDetectionHashEngineBuilder builder = + new DeviceDetectionHashEngineBuilder(loggerFactory, null) + .setPerformanceProfile( + Constants.PerformanceProfiles.HighPerformance) + .setAutoUpdate(false) + .setDataFileSystemWatcher(false); + if (names != null) { + builder.setProperties(names); + } + try (DeviceDetectionHashEngine engine = + builder.build(file.toString(), false)) { + Field field = DeviceDetectionHashEngine.class + .getDeclaredField("propertiesConfigSwig"); + field.setAccessible(true); + RequiredPropertiesConfigSwig config = + (RequiredPropertiesConfigSwig) field.get(engine); + try (VectorStringSwig given = config.getProperties()) { + return new ArrayList<>(given); + } + } + } + + @Test + public void NoNames_EngineCreatedWithNoNames() throws Exception { + File file = liteFile(); + List names = namesCreatedWith(file, null); + assertTrue("The engine was created with names " + names + + " when none were configured", names.isEmpty()); + } + + @Test + public void Names_EngineCreatedWithThoseNames() throws Exception { + File file = liteFile(); + List configured = Arrays.asList("IsMobile", "BrowserName"); + assertEquals(configured, namesCreatedWith(file, configured)); + } +} diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ProfilesForPropertyValueTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ProfilesForPropertyValueTests.java new file mode 100644 index 000000000..a7f9926af --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ProfilesForPropertyValueTests.java @@ -0,0 +1,141 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.VectorUInt32Swig; +import fiftyone.pipeline.engines.Constants; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +import static fiftyone.devicedetection.shared.testhelpers.TestFileUtils.LITE_HASH_DATA_FILE_PATH; +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The Engine MUST be able to give the ids of the profiles that hold a given + * value of a given Property, such as a TAC or a native model name, through + * the native code's EngineHash::getProfileIdsForPropertyValue() + * (device-detection-specification/pipeline-elements/device-detection-on-premise.md + * line 479 at 706de1f0). The bullets after it say the name and the value are + * matched exactly, the ids are ascending and each appears once, and the + * Indexed Properties option gives the same ids without reading every + * profile. + */ +public class ProfilesForPropertyValueTests { + + private static final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + /** + * The engine gives the ids the native function gives, in its order, and + * passes the name and the value to it as they were given. + */ + @Test + public void ProfilesForValue_FromTheNativeFunction() throws Exception { + EngineOverMockNativeCode mocks = new EngineOverMockNativeCode(); + VectorUInt32Swig ids = mock(VectorUInt32Swig.class); + when(ids.size()).thenReturn(3); + when(ids.get(0)).thenReturn(12280L); + when(ids.get(1)).thenReturn(17470L); + when(ids.get(2)).thenReturn(4294967295L); + when(mocks.nativeEngine.getProfileIdsForPropertyValue( + "NativeModel", "iPhone14,2")).thenReturn(ids); + + List result = mocks.engine.getProfileIdsForPropertyValue( + "NativeModel", "iPhone14,2"); + + assertEquals(Arrays.asList(12280L, 17470L, 4294967295L), result); + verify(mocks.nativeEngine).getProfileIdsForPropertyValue( + "NativeModel", "iPhone14,2"); + // The native vector is released once its ids are copied. + verify(ids).close(); + } + + private static File liteFile() { + try { + return getFilePath(LITE_HASH_DATA_FILE_PATH); + } + catch (IllegalArgumentException e) { + assumeTrue(e.getMessage(), false); + return null; + } + } + + private static DeviceDetectionHashEngine build( + File file, List indexed) throws Exception { + DeviceDetectionHashEngineBuilder builder = + new DeviceDetectionHashEngineBuilder(loggerFactory, null) + .setPerformanceProfile( + Constants.PerformanceProfiles.HighPerformance) + .setAutoUpdate(false) + .setDataFileSystemWatcher(false); + if (indexed != null) { + builder.setIndexedProperties(indexed); + } + return builder.build(file.toString(), false); + } + + /** + * Against the free data file: ids are ascending and distinct, a value in + * another case gives none, and indexing the Property gives the same ids. + */ + @Test + public void ProfilesForValue_LiteDataFile() throws Exception { + File file = liteFile(); + List unindexed; + List otherCase; + try (DeviceDetectionHashEngine engine = build(file, null)) { + unindexed = engine.getProfileIdsForPropertyValue( + "PlatformName", "iOS"); + otherCase = engine.getProfileIdsForPropertyValue( + "PlatformName", "IOS"); + } + assertFalse("No profiles hold PlatformName iOS", unindexed.isEmpty()); + List sorted = new ArrayList<>(unindexed); + Collections.sort(sorted); + assertEquals("Ids are not ascending", sorted, unindexed); + assertEquals("An id appears more than once", + unindexed.size(), new HashSet<>(unindexed).size()); + assertTrue("A value in another case matched", otherCase.isEmpty()); + + try (DeviceDetectionHashEngine engine = + build(file, Collections.singletonList("PlatformName"))) { + assertEquals(unindexed, engine.getProfileIdsForPropertyValue( + "PlatformName", "iOS")); + } + } +} diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java index 0302a3a95..a58c909cc 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java @@ -53,8 +53,8 @@ import java.util.Map; import java.util.Set; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.TAC_HASH_DATA_FILE_NAME; +import static fiftyone.devicedetection.shared.testhelpers.TestFileUtils.LITE_HASH_DATA_FILE_PATH; +import static fiftyone.devicedetection.shared.testhelpers.TestFileUtils.TAC_HASH_DATA_FILE_NAME; import static fiftyone.pipeline.util.FileFinder.getFilePath; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyMetaDataTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyMetaDataTests.java new file mode 100644 index 000000000..edd639ed9 --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyMetaDataTests.java @@ -0,0 +1,102 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.File; + +import static fiftyone.devicedetection.shared.testhelpers.TestFileUtils.LITE_HASH_DATA_FILE_PATH; +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +/** + * Every Property of the Hash engine has the metadata an Aspect Engine must + * add to that of a Flow Element: a description, and the data tiers where the + * Property is present. + *

+ * Specification: pipeline-specification/features/properties.md line 101 at + * 51Degrees/specifications 706de1f0, "Aspect Engines are REQUIRED to + * populate some additional metadata values beyond those described for Flow + * Elements.", with lines 108 and 109 naming Description and Data tiers where + * present. + */ +public class PropertyMetaDataTests { + + @Test + public void everyProperty_HasDescriptionAndDataTiers() throws Exception { + File dataFile = null; + try { + dataFile = getFilePath(LITE_HASH_DATA_FILE_PATH); + } catch (IllegalArgumentException e) { + assumeTrue(e.getMessage(), false); + } + + try (DeviceDetectionHashEngine engine = + new DeviceDetectionHashEngineBuilder( + LoggerFactory.getILoggerFactory(), null) + .setPerformanceProfile( + Constants.PerformanceProfiles.LowMemory) + .setAutoUpdate(false) + .build(dataFile.toString(), false)) { + String tier = engine.getDataSourceTier(); + assertFalse(engine.getProperties().isEmpty()); + for (FiftyOneAspectPropertyMetaData property : + engine.getProperties()) { + String name = property.getName(); + assertNotNull(name + " has a description", + property.getDescription()); + assertFalse(name + " has a description", + property.getDescription().trim().isEmpty()); + assertNotNull(name + " has data tiers", + property.getDataTiersWherePresent()); + assertFalse(name + " has data tiers", + property.getDataTiersWherePresent().isEmpty()); + if (property.isAvailable()) { + assertTrue( + name + " is in the data file, so its tiers name " + + "the data file's tier '" + tier + "'", + containsIgnoringCase( + property.getDataTiersWherePresent(), tier)); + } + } + } + } + + private static boolean containsIgnoringCase( + Iterable values, + String value) { + for (String candidate : values) { + if (candidate.equalsIgnoreCase(value)) { + return true; + } + } + return false; + } +} diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ReadByPositionTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ReadByPositionTests.java new file mode 100644 index 000000000..f26a0a81e --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ReadByPositionTests.java @@ -0,0 +1,124 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.devicedetection.hash.engine.onpremise.TestsBase; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.BoolValueSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.ResultsHashSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.StringValueSwig; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.*; + +/** + * A read of a property the engine has a position for goes by that + * position, and asks the native results nothing by name, not even whether + * they hold the property. + */ +public class ReadByPositionTests extends TestsBase { + + private static final Logger logger = mock(Logger.class); + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.HighPerformance); + } + + @After + public void cleanup() { + testCleanup(); + } + + @Test + public void BooleanRead_ByPosition_NoLookupByName() throws Exception { + DeviceDetectionHashEngine engine = getWrapper().getEngine(); + Integer index = engine.getPropertyIndex("IsMobile"); + assertNotNull(index); + try (FlowData flowData = + getWrapper().getPipeline().createFlowData()) { + ResultsHashSwig results = results(); + BoolValueSwig value = mock(BoolValueSwig.class); + when(value.hasValue()).thenReturn(true); + when(value.getValue()).thenReturn(true); + when(results.getValueAsBool(index.intValue())).thenReturn(value); + DeviceDataHashDefault data = data(flowData, engine, results); + + assertEquals(Boolean.TRUE, data.getIsMobile().getValue()); + + verify(results).getValueAsBool(index.intValue()); + verify(results, never()).containsProperty(any(byte[].class)); + } + } + + @Test + public void StringRead_ByPosition_NoLookupByName() throws Exception { + DeviceDetectionHashEngine engine = getWrapper().getEngine(); + Integer index = engine.getPropertyIndex("BrowserName"); + assertNotNull(index); + try (FlowData flowData = + getWrapper().getPipeline().createFlowData()) { + ResultsHashSwig results = results(); + StringValueSwig value = mock(StringValueSwig.class); + when(value.hasValue()).thenReturn(true); + when(value.getValue()).thenReturn("Safari"); + when(results.getValueAsString(index.intValue())).thenReturn(value); + DeviceDataHashDefault data = data(flowData, engine, results); + + assertEquals("Safari", data.getBrowserName().getValue()); + + verify(results).getValueAsString(index.intValue()); + verify(results, never()).containsProperty(any(byte[].class)); + } + } + + /** + * Native results that say they hold every property, so that a read + * which asks by name still reaches the value. + */ + private static ResultsHashSwig results() { + ResultsHashSwig results = mock(ResultsHashSwig.class); + when(results.containsProperty(any(byte[].class))).thenReturn(true); + return results; + } + + private static DeviceDataHashDefault data( + FlowData flowData, + DeviceDetectionHashEngine engine, + ResultsHashSwig results) { + DeviceDataHashDefault data = new DeviceDataHashDefault( + logger, + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + data.setResults(results); + return data; + } +} diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ReadOnlyWhenRequestedTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ReadOnlyWhenRequestedTests.java new file mode 100644 index 000000000..28240f638 --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ReadOnlyWhenRequestedTests.java @@ -0,0 +1,123 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.devicedetection.hash.engine.onpremise.interop.Swig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.BoolValueSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.EvidenceDeviceDetectionSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.ResultsHashSwig; +import fiftyone.pipeline.core.data.Evidence; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.services.MissingPropertyService; +import org.junit.Test; +import org.slf4j.Logger; + +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.AdditionalMatchers.aryEq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +/** + * The Element Data MUST read a Property value from ResultsHash only when + * that Property is requested, and MUST NOT copy Property values out of + * ResultsHash while the Engine processes the Flow Data + * (device-detection-specification/pipeline-elements/device-detection-on-premise.md + * line 152 at 706de1f0). The native results here are mocks that record + * every call made to them. + */ +public class ReadOnlyWhenRequestedTests { + + private static DeviceDataHashDefault newData( + DeviceDetectionHashEngine engine) { + return new DeviceDataHashDefault( + mock(Logger.class), + mock(FlowData.class), + engine, + mock(MissingPropertyService.class)); + } + + /** + * Processing reads nothing from the results it stores. This creates + * native evidence, so it needs the native library, though no data file. + */ + @Test + public void Process_ReadsNoValue() throws Exception { + EngineOverMockNativeCode mocks = new EngineOverMockNativeCode(); + Field keys = DeviceDetectionHashEngine.class + .getDeclaredField("evidenceKeys"); + keys.setAccessible(true); + keys.set(mocks.engine, + Collections.singletonList("header.user-agent")); + ResultsHashSwig results = mock(ResultsHashSwig.class); + when(mocks.nativeEngine.process(any(EvidenceDeviceDetectionSwig.class))) + .thenReturn(results); + + Map evidence = new HashMap<>(); + evidence.put("header.user-agent", "Mozilla/5.0"); + Evidence flowEvidence = mock(Evidence.class); + when(flowEvidence.asKeyMap()).thenReturn(evidence); + FlowData flowData = mock(FlowData.class); + when(flowData.getEvidence()).thenReturn(flowEvidence); + + DeviceDataHashDefault data = newData(mocks.engine); + mocks.engine.processEngine(flowData, data); + + verify(mocks.nativeEngine).process(any(EvidenceDeviceDetectionSwig.class)); + verifyNoInteractions(results); + } + + /** + * Storing the results reads nothing, and a request for one Property reads + * that Property and no other. + */ + @Test + public void Request_ReadsOnlyThatProperty() throws Exception { + EngineOverMockNativeCode mocks = new EngineOverMockNativeCode(); + ResultsHashSwig results = mock(ResultsHashSwig.class); + BoolValueSwig value = mock(BoolValueSwig.class); + when(value.hasValue()).thenReturn(true); + when(value.getValue()).thenReturn(true); + when(results.containsProperty(any(byte[].class))).thenReturn(true); + when(results.getValueAsBool(any(byte[].class))).thenReturn(value); + when(results.getValueAsBool(any(int.class))).thenReturn(value); + + DeviceDataHashDefault data = newData(mocks.engine); + data.setResults(results); + verifyNoInteractions(results); + + assertTrue(data.getIsMobile().getValue()); + byte[] name = Swig.asBytes("ismobile"); + verify(results).containsProperty(aryEq(name)); + verify(results).getValueAsBool(aryEq(name)); + verifyNoMoreInteractions(results); + } +} diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ReadsByPositionTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ReadsByPositionTests.java new file mode 100644 index 000000000..ec5f938a0 --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ReadsByPositionTests.java @@ -0,0 +1,73 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.BoolValueSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.ResultsHashSwig; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.services.MissingPropertyService; +import org.junit.Test; +import org.slf4j.Logger; + +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +/** + * Every read SHOULD use the position + * (device-detection-specification/pipeline-elements/device-detection-on-premise.md + * line 205 at 706de1f0). A read by name looks the name up in the table from + * the names the engine was created with to their positions, and then reads + * by position. The native results here are mocks that record every call. + */ +public class ReadsByPositionTests { + + @Test + public void Read_OfANamedProperty_IsByPositionOnly() throws Exception { + EngineOverMockNativeCode mocks = new EngineOverMockNativeCode(); + mocks.setPropertyNames("ScreenPixelsHeight", "IsMobile"); + + ResultsHashSwig results = mock(ResultsHashSwig.class); + BoolValueSwig value = mock(BoolValueSwig.class); + when(value.hasValue()).thenReturn(true); + when(value.getValue()).thenReturn(true); + when(results.containsProperty(any(byte[].class))).thenReturn(true); + when(results.getValueAsBool(any(byte[].class))).thenReturn(value); + when(results.getValueAsBool(1)).thenReturn(value); + + DeviceDataHashDefault data = new DeviceDataHashDefault( + mock(Logger.class), + mock(FlowData.class), + mocks.engine, + mock(MissingPropertyService.class)); + data.setResults(results); + + assertTrue(data.getIsMobile().getValue()); + // IsMobile is at position 1, and nothing is read by its name. + verify(results).getValueAsBool(1); + verifyNoMoreInteractions(results); + } +} diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ResultsNotLockedTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ResultsNotLockedTests.java new file mode 100644 index 000000000..0dde750cd --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/ResultsNotLockedTests.java @@ -0,0 +1,147 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.ResultsHashSwig; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.services.MissingPropertyService; +import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.RETURNS_DEFAULTS; +import static org.mockito.Mockito.mock; + +/** + * Aspect Data that reads its values from a ResultsHash instance MUST NOT lock + * it, and the caller is responsible for thread-safe access. Each test reads + * the same data from two threads, and the mock results record the most + * threads that were ever inside them at once. A thread that gets in waits for + * the other to join it, up to a limit. With no lock both are inside together. + * With one the other cannot join, the wait runs out, and one thread was + * inside at a time. + * @see Specification + */ +public class ResultsNotLockedTests { + + private static final long JOIN_WAIT_MILLIS = 5000; + + /** + * Answer for every native results call that counts the threads inside the + * results, and gives a value of the type the call returns. + */ + private static class CountingAnswer implements Answer { + private final AtomicInteger inside = new AtomicInteger(); + private final AtomicInteger most = new AtomicInteger(); + private final CountDownLatch bothInside = new CountDownLatch(2); + + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + String name = invocation.getMethod().getName(); + if (name.equals("close") || name.equals("delete") || + name.equals("finalize")) { + return null; + } + int now = inside.incrementAndGet(); + most.accumulateAndGet(now, Math::max); + bothInside.countDown(); + try { + bothInside.await(JOIN_WAIT_MILLIS, TimeUnit.MILLISECONDS); + Class returns = invocation.getMethod().getReturnType(); + if (name.equals("containsProperty")) { + return true; + } + if (returns.getName().contains(".interop.swig.")) { + // A value with no value, as the native code gives for a + // property with none. + return mock(returns); + } + return RETURNS_DEFAULTS.answer(invocation); + } + finally { + inside.decrementAndGet(); + } + } + } + + private static int mostThreadsInsideResults( + Consumer read) throws Exception { + CountingAnswer answer = new CountingAnswer(); + ResultsHashSwig results = mock(ResultsHashSwig.class, answer); + FlowData flowData = mock(FlowData.class); + DeviceDataHashDefault data = new DeviceDataHashDefault( + mock(Logger.class), + flowData, + mock(DeviceDetectionHashEngine.class), + mock(MissingPropertyService.class)); + data.setResults(results); + + CountDownLatch start = new CountDownLatch(1); + List errors = new ArrayList<>(); + List threads = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + Thread thread = new Thread(() -> { + try { + start.await(); + read.accept(data); + } catch (Throwable t) { + synchronized (errors) { + errors.add(t); + } + } + }); + thread.start(); + threads.add(thread); + } + start.countDown(); + for (Thread thread : threads) { + thread.join(); + } + if (errors.isEmpty() == false) { + throw new AssertionError("A read failed", errors.get(0)); + } + return answer.most.get(); + } + + /** + * The public list read goes to the native results with no lock, so two + * threads that share the data are inside the results together. + */ + @Test + public void Results_GetValues_NotLocked() throws Exception { + assertEquals( + "One thread waited for the other to leave the native results " + + "for getValues, so the read took a lock", + 2, + mostThreadsInsideResults(d -> d.getValues("HardwareName"))); + } +} diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/TestProcessTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/TestProcessTests.java index 46b43db96..768e4eb5b 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/TestProcessTests.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/TestProcessTests.java @@ -67,6 +67,11 @@ public void Process_Hash_CaseInsensitiveKeys() throws Exception { ProcessTests.caseInsensitiveEvidenceKeys(getWrapper(), new DataValidatorHash(getWrapper().getEngine())); } + @Test + public void Process_Hash_CaseInsensitivePrefix() throws Exception { + ProcessTests.caseInsensitiveEvidencePrefix(getWrapper(), new DataValidatorHash(getWrapper().getEngine())); + } + @Test public void Process_Hash_ProfileOverride() throws Exception { ProcessTests.profileOverride(getWrapper(), new DataValidatorHash(getWrapper().getEngine())); diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/WritesAfterProcessingTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/WritesAfterProcessingTests.java new file mode 100644 index 000000000..67bce5bcd --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/WritesAfterProcessingTests.java @@ -0,0 +1,103 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.devicedetection.hash.engine.onpremise.TestsBase; +import fiftyone.devicedetection.shared.DeviceData; +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.data.AspectDataBase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.function.ThrowingRunnable; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; + +/** + * The check, for the Flow Element in this module, the + * {@link DeviceDetectionHashEngine}, that processes it, then writes to its + * Element Data and expects the error. Values the Element Data keeps once it + * has read them from the results are not writes, so reading every value + * still works. + * @see Specification + */ +public class WritesAfterProcessingTests extends TestsBase { + + private static final String MOBILE = + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_1 like Mac OS X) " + + "AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 " + + "Mobile/11D167 Safari/9537.53"; + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.HighPerformance); + } + + @After + public void cleanup() { + testCleanup(); + } + + private static void assertFails(String what, ThrowingRunnable write) { + assertThrows(what, IllegalStateException.class, write); + } + + /** + * Every write method of Element Data, and of Aspect Data, fails once its + * Flow Element has finished. + * @param data the Element Data of a finished Flow Element + */ + static void assertWritesFail(ElementData data) { + assertFails("put", () -> data.put("after", "value")); + assertFails("populateFromMap", () -> data.populateFromMap( + Collections.singletonMap("after", "value"))); + if (data instanceof AspectDataBase) { + AspectDataBase aspect = (AspectDataBase) data; + assertFails("setCloudRequestFailed", aspect::setCloudRequestFailed); + assertFails("addEngine", () -> aspect.addEngine(null)); + assertFails("addProcessCallable", + () -> aspect.addProcessCallable(null)); + } + } + + @Test + public void deviceDetectionHashEngine_ElementDataRefusesWrites() + throws Exception { + try (FlowData data = getWrapper().getPipeline().createFlowData()) { + data.addEvidence("header.user-agent", MOBILE); + data.process(); + assertEquals(String.valueOf(data.getErrors()), + 0, data.getErrors().size()); + + DeviceData own = data.getFromElement(getWrapper().getEngine()); + assertFalse(own.asKeyMap().isEmpty()); + assertWritesFail(own); + } + } +} diff --git a/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/DeviceDataBaseOnPremise.java b/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/DeviceDataBaseOnPremise.java index fcf900ce6..ac69c3bf2 100644 --- a/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/DeviceDataBaseOnPremise.java +++ b/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/DeviceDataBaseOnPremise.java @@ -29,11 +29,13 @@ import fiftyone.pipeline.engines.data.AspectData; import fiftyone.pipeline.engines.data.AspectPropertyMetaData; import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.data.ProvidesValuesAsString; import fiftyone.pipeline.engines.flowelements.AspectEngine; import fiftyone.pipeline.engines.services.MissingPropertyService; import org.slf4j.Logger; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.TreeMap; @@ -41,8 +43,15 @@ /** * Base class used for all 51Degrees on-premise device detection results classes. + *

+ * An instance reads its values from native results, and reading a value + * writes into them, so an instance is not safe to use from several threads + * at once. The caller is responsible for thread-safe access, for example by + * wrapping its calls in a lock of its own. See the + * thread safety specification. */ -public abstract class DeviceDataBaseOnPremise extends DeviceDataBase { +public abstract class DeviceDataBaseOnPremise extends DeviceDataBase + implements ProvidesValuesAsString { protected final Map, Class> primitiveTypes; private final Object dataLock = new Object(); @@ -93,6 +102,27 @@ public abstract AspectPropertyValue> getValues( protected abstract AspectPropertyValue getValueAsString( String propertyName); + /** + * {@inheritDoc} + *

+ * The Property is first read through its typed accessor, so a Property + * that is not in the results throws, and a lazy loaded result is waited + * for, exactly as for that accessor. + *

+ * The value is read from the results instance with no lock, so this + * instance is not safe to use from several threads at once. A caller + * that shares it coordinates its own access. + */ + @SuppressWarnings("unchecked") + @Override + public AspectPropertyValue getStoredValueAsString( + String propertyName) { + getAs(propertyName, + AspectPropertyValue.class, + getPropertyType(propertyName)); + return getValueAsString(propertyName); + } + /** * Get the value for the specified property as an {@link Integer}. If the * property cannot be represented as an {@link Integer}, then the result @@ -147,12 +177,12 @@ public Map asKeyMap() { for (ElementPropertyMetaData property : getPipeline() .getElementAvailableProperties() .get(getEngines().get(0).getElementDataKey()).values()) { - map.put(property.getName().toLowerCase(), + map.put(property.getName().toLowerCase(Locale.ROOT), getAs(property.getName(), AspectPropertyValue.class, property.getType())); } - populateFromMap(map); + keepValues(map); mapPopulated = true; } } diff --git a/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/FileUtils.java b/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/FileUtils.java index fe9e13a9e..aa5b01ddf 100644 --- a/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/FileUtils.java +++ b/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/FileUtils.java @@ -36,6 +36,14 @@ import static fiftyone.pipeline.util.FileFinder.getFilePath; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +/** + * Finds the data and evidence files the tests use. + * @deprecated These are test helpers, which the published package should + * not hold. Example code takes the data file names from + * {@code fiftyone.devicedetection.examples.shared.DataFileHelper}, in the + * device-detection-java-examples.shared module. + */ +@Deprecated @SuppressWarnings("OptionalUsedAsFieldOrParameterType") public class FileUtils { public static final String ENTERPRISE_HASH_DATA_FILE_NAME = "Enterprise-HashV41.hash"; diff --git a/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/KeyUtils.java b/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/KeyUtils.java index fdc8d2d6d..ee29b5f69 100644 --- a/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/KeyUtils.java +++ b/device-detection/device-detection.shared/src/main/java/fiftyone/devicedetection/shared/testhelpers/KeyUtils.java @@ -23,11 +23,17 @@ package fiftyone.devicedetection.shared.testhelpers; import java.util.Base64; +import java.util.Locale; import java.util.Objects; /** * Helpers to obtain keys from the environment + * @deprecated These are test helpers, which the published package should + * not hold. Example code reads and checks keys with + * {@code fiftyone.devicedetection.examples.shared.KeyHelper}, in the + * device-detection-java-examples.shared module. */ +@Deprecated public class KeyUtils { /** * Obtain a key either from environment variable or from a property. @@ -37,7 +43,7 @@ public class KeyUtils { public static String getNamedKey(String keyName) { String resourceKey = System.getenv(keyName); if (Objects.isNull(resourceKey)) { - resourceKey = System.getenv(keyName.toUpperCase()); + resourceKey = System.getenv(keyName.toUpperCase(Locale.ROOT)); if (Objects.isNull(resourceKey)) { resourceKey = System.getProperty(keyName); } diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/PublishedPackageTests.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/PublishedPackageTests.java new file mode 100644 index 000000000..6412f2651 --- /dev/null +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/PublishedPackageTests.java @@ -0,0 +1,101 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.shared; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertTrue; + +/** + * The published Device Detection packages hold no tests or test helpers, + * other than deprecated test helpers that earlier releases published, which + * stay because a patch release removes no public name. Each module's main + * sources are what its published jar holds, so they are read for a test + * helper package or a test class. + */ +public class PublishedPackageTests { + + /** + * The published Device Detection modules, which sit beside this one. + */ + private static final String[] MODULES = { + "device-detection.shared", + "device-detection.cloud", + "device-detection.hash.engine.on-premise", + "device-detection" + }; + + @Test + public void publishedModules_HoldNoTestCode() throws IOException { + Path root = Paths.get(System.getProperty("basedir", ".")) + .toAbsolutePath() + .getParent(); + List found = new ArrayList<>(); + for (String module : MODULES) { + Path sources = root.resolve(module) + .resolve("src").resolve("main").resolve("java"); + assertTrue(sources + " should hold the module's sources", + Files.isDirectory(sources)); + List files; + try (Stream walk = Files.walk(sources)) { + files = walk + .filter(p -> p.toString().endsWith(".java")) + .map(p -> sources.relativize(p).toString() + .replace('\\', '/')) + .collect(Collectors.toList()); + } + for (String file : files) { + if ((file.contains("/testhelpers/") && + isDeprecated(file) == false) || + file.matches(".*Tests?\\.java")) { + found.add(module + ": " + file); + } + } + } + assertTrue("These published sources are tests or test helpers: " + + found, found.isEmpty()); + } + + /** + * Whether the class a source file holds is deprecated, which is false + * for a class this module cannot load. + */ + private static boolean isDeprecated(String file) { + String name = file.substring(0, file.length() - ".java".length()) + .replace('/', '.'); + try { + return Class.forName(name).isAnnotationPresent(Deprecated.class); + } catch (ClassNotFoundException e) { + return false; + } + } +} diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/TestHelpersStayPublishedTests.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/TestHelpersStayPublishedTests.java new file mode 100644 index 000000000..2eb8094e1 --- /dev/null +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/TestHelpersStayPublishedTests.java @@ -0,0 +1,62 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.shared; + +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * FileUtils and KeyUtils, which earlier releases published, stay in the + * published jar, deprecated, as a patch release removes no public name. + * They are found by name, because the build treats the use of a + * deprecated class as an error. + */ +public class TestHelpersStayPublishedTests { + + private static final String PACKAGE = + "fiftyone.devicedetection.shared.testhelpers."; + + @Test + public void FileUtils_InTheMainJar_Deprecated() throws Exception { + assertPublishedAndDeprecated(PACKAGE + "FileUtils"); + } + + @Test + public void KeyUtils_InTheMainJar_Deprecated() throws Exception { + assertPublishedAndDeprecated(PACKAGE + "KeyUtils"); + } + + private static void assertPublishedAndDeprecated(String name) + throws ClassNotFoundException { + Class type = Class.forName(name); + String location = type.getProtectionDomain() + .getCodeSource().getLocation().toString(); + assertFalse(name + " comes from " + location + ", not the main jar", + location.contains("test-classes") || + location.endsWith("-tests.jar")); + assertTrue(name + " should be deprecated", + type.isAnnotationPresent(Deprecated.class)); + } +} diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/Constants.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/Constants.java index 4b48d6c91..d7e8b5665 100644 --- a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/Constants.java +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/Constants.java @@ -51,5 +51,5 @@ public class Constants { public static final String ChromeUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"; - public static final String RESOURCE_KEY_ENV_VAR = "TestResourceKey"; + public static final String RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY"; } diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/FileUtilsTest.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/FileUtilsTest.java index d6cef9ea9..16a3d9845 100644 --- a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/FileUtilsTest.java +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/FileUtilsTest.java @@ -34,7 +34,7 @@ public class FileUtilsTest { - Logger logger = LoggerFactory.getLogger(FileUtils.class); + Logger logger = LoggerFactory.getLogger(TestFileUtils.class); /** * location of "." is the same as user.dir diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/KeyUtilsTest.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/KeyUtilsTest.java index ce847992b..1b9f01bfe 100644 --- a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/KeyUtilsTest.java +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/KeyUtilsTest.java @@ -29,8 +29,8 @@ public class KeyUtilsTest { @Test public void testGetEnv(){ - assertNotNull(KeyUtils.getNamedKey("path")); - System.out.println(KeyUtils.getNamedKey("path")); + assertNotNull(TestKeyUtils.getNamedKey("path")); + System.out.println(TestKeyUtils.getNamedKey("path")); } } \ No newline at end of file diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/TestFileUtils.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/TestFileUtils.java new file mode 100644 index 000000000..63a51de8a --- /dev/null +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/TestFileUtils.java @@ -0,0 +1,169 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.shared.testhelpers; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; +import java.util.Optional; + +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +/** + * Finds the data and evidence files the tests use. + */ +@SuppressWarnings("OptionalUsedAsFieldOrParameterType") +public class TestFileUtils { + public static final String ENTERPRISE_HASH_DATA_FILE_NAME = "Enterprise-HashV41.hash"; + + public static final String TAC_HASH_DATA_FILE_NAME = "TAC-HashV41.hash"; + public static final String LITE_HASH_DATA_FILE_NAME = "51Degrees-LiteV4.1.hash"; + private static Optional HASH_DATA_FILE; + public static final String UA_FILE_NAME = "20000 User Agents.csv"; + private static File UA_FILE; + public static final String EVIDENCE_FILE_NAME = "20000 Evidence Records.yml"; + private static Optional EVIDENCE_FILE; + // Where each free file is in the data repository. A search for the path + // rather than the name does not stop at another file of the same name, + // such as a Git LFS pointer in an old device-detection-data folder. + public static final String LITE_HASH_DATA_FILE_PATH = + "device-detection/" + LITE_HASH_DATA_FILE_NAME; + public static final String UA_FILE_PATH = "device-detection/" + UA_FILE_NAME; + public static final String EVIDENCE_FILE_PATH = + "device-detection/" + EVIDENCE_FILE_NAME; + + /** + * Helper to find the location of an Enterprise or Lite Hash file in the default search scope + * + * @return a file or null if not found + */ + public static String getHashFileName() { + return Objects.isNull(getHashFile()) ? null : getHashFile().getPath(); + } + + /** + * Helper to find the location of an Enterprise or Lite Hash file in the default search scope + * + * @return a file or empty if not found + */ + public static File getHashFile() { + try { + if (Objects.nonNull(HASH_DATA_FILE)){ + return HASH_DATA_FILE.orElse(null); + } + // following throws exception if not found + HASH_DATA_FILE = Optional.of(getFilePath(TAC_HASH_DATA_FILE_NAME)); + return HASH_DATA_FILE.get(); + } catch (Exception e) { + try { + // following throws exception if not found + HASH_DATA_FILE = Optional.of(getFilePath(ENTERPRISE_HASH_DATA_FILE_NAME)); + return HASH_DATA_FILE.get(); + } catch (Exception ex) { + try { + // following throws exception if not found + HASH_DATA_FILE = Optional.of(getFilePath(LITE_HASH_DATA_FILE_PATH)); + return HASH_DATA_FILE.get(); + } catch (Exception exc) { + HASH_DATA_FILE = Optional.empty(); + return null; + } + } + } + } + + public static File getEvidenceFile() { + if (Objects.nonNull(EVIDENCE_FILE)) { + return EVIDENCE_FILE.orElse(null); + } + try { + EVIDENCE_FILE = Optional.of(getFilePath(EVIDENCE_FILE_PATH)); + return EVIDENCE_FILE.get(); + } catch (Exception e) { + EVIDENCE_FILE = Optional.empty(); + return null; + } + } + + /** + * Prefix for temp files that are created by {@link #jarFileHelper(String)} + **/ + public static String TEMP_FILE_PREFIX = "DDTempFile"; + + /** + * Search the classpath for a resource. If it doesn't exist throw an exception. + * If the file is in a jar then copy it to a temp file, so it can be used as an actual file. + * + * Callers might wish to delete the temp file created when one is created and + * can assess whether this is the case as the name of the filename created will + * start with {@link #TEMP_FILE_PREFIX} which they can alter to suit their needs. + * + * Files created here have the property deleteOnExit; + * + * @param file a filename + * @return a File representing the resource + */ + public static File jarFileHelper(String file) { + // find the resource in the classpath + URL resource = Objects.requireNonNull(TestFileUtils.class.getClassLoader().getResource(file), + file + " could not be found on the classpath"); + try { + return new File(resource.toURI()); + } catch (Exception e) { + if (!resource.getPath().contains(".jar!")) + throw new IllegalStateException(resource.getPath() + " can't be loaded", e); + } + // make a copy of the file if it is in a jar file (assumption is that if the + // name matches the pattern then it is in a jar, if not, no harm is done + // but an irrelevant tmp file may be created) + String filePath; + try { + filePath = URLDecoder.decode(resource.getPath(), "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException(e); + } + // we will try to match the extension + int index = filePath.lastIndexOf("."); + String extension = index < 0 ? "tmp" : filePath.substring(index); + try { + Path temp = Files.createTempFile(TEMP_FILE_PREFIX, extension); + try (InputStream is = TestFileUtils.class.getClassLoader().getResourceAsStream(file)) { + //noinspection ConstantConditions + Files.copy(is, temp, REPLACE_EXISTING); + } + filePath = temp.toAbsolutePath().toString(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + File f = new File(filePath); + f.deleteOnExit(); + return f; + } +} \ No newline at end of file diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/TestKeyUtils.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/TestKeyUtils.java new file mode 100644 index 000000000..678cdcfc5 --- /dev/null +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/TestKeyUtils.java @@ -0,0 +1,63 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.shared.testhelpers; + +import java.util.Base64; +import java.util.Objects; + +/** + * Helpers to obtain keys from the environment + */ +public class TestKeyUtils { + /** + * Obtain a key either from environment variable or from a property. + *

+ * Try resource key as env var, then as upper case env var, the system property + */ + public static String getNamedKey(String keyName) { + String resourceKey = System.getenv(keyName); + if (Objects.isNull(resourceKey)) { + resourceKey = System.getenv(keyName.toUpperCase()); + if (Objects.isNull(resourceKey)) { + resourceKey = System.getProperty(keyName); + } + } + return resourceKey; + } + + /** + * Evaluate whether a key might be valid + * @param keyValue value to test + * @return boolean + */ + public static boolean isInvalidKey(String keyValue){ + Base64.Decoder decoder = Base64.getUrlDecoder(); + try { + return Objects.isNull(keyValue) || + keyValue.trim().length() < 19 || + decoder.decode(keyValue).length < 14; + } catch (IllegalArgumentException e) { + return true; + } + } +} diff --git a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/flowelements/ProcessTests.java b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/flowelements/ProcessTests.java index b0746c22e..97f2f2f89 100644 --- a/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/flowelements/ProcessTests.java +++ b/device-detection/device-detection.shared/src/test/java/fiftyone/devicedetection/shared/testhelpers/flowelements/ProcessTests.java @@ -87,6 +87,21 @@ public static void caseInsensitiveEvidenceKeys( } } + /** + * Evidence keys ignore case, the prefix as well as the field, so a + * User-Agent given with its prefix in capitals is used. + */ + public static void caseInsensitiveEvidencePrefix( + Wrapper wrapper, + DataValidator validator) throws Exception { + try (FlowData data = wrapper.getPipeline().createFlowData()) { + data + .addEvidence("HEADER.User-Agent", Constants.ChromeUserAgent) + .process(); + validator.validateData(data, true); + } + } + public static void profileOverride( Wrapper wrapper, DataValidator validator) throws Exception { diff --git a/device-detection/device-detection/pom.xml b/device-detection/device-detection/pom.xml index 96eefb171..2608c4e91 100644 --- a/device-detection/device-detection/pom.xml +++ b/device-detection/device-detection/pom.xml @@ -12,7 +12,7 @@ device-detection 51Degrees :: Device Detection - Parse HTTP headers to detect the device type, model, OS, browser, and crawler information. This is an alternative to popular UAParser, DeviceAtlas, and WURFL packages. + Device Detection and User-Agent parser for Java. Detects the device type and model, browser, operating system, and crawlers from the User-Agent and User-Agent Client Hints. Works on-premise with a data file or in the cloud. Built on the 51Degrees Pipeline API. An alternative to UAParser.js, DeviceAtlas, and WURFL. https://github.com/51Degrees/51degrees-java diff --git a/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionCloudPipelineBuilder.java b/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionCloudPipelineBuilder.java index 9d1e2ecfc..a063a588d 100644 --- a/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionCloudPipelineBuilder.java +++ b/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionCloudPipelineBuilder.java @@ -93,6 +93,10 @@ public Pipeline build() throws Exception { if (licenseKey != null && licenseKey.isEmpty() == false) { cloudRequestEngineBuilder.setLicenseKey(licenseKey); } + if (requestedProperties.isEmpty() == false) { + cloudRequestEngineBuilder.setRequestedProperties( + requestedProperties); + } if (cloudRequestOrigin != null && cloudRequestOrigin.isEmpty() == false) { cloudRequestEngineBuilder.setCloudRequestOrigin(cloudRequestOrigin); } diff --git a/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilder.java b/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilder.java index 63d98b785..f38644676 100644 --- a/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilder.java +++ b/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilder.java @@ -24,7 +24,9 @@ import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngineBuilder; +import fiftyone.pipeline.annotations.DefaultValue; import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.Constants; import fiftyone.pipeline.engines.data.AspectData; @@ -48,7 +50,9 @@ public class DeviceDetectionOnPremisePipelineBuilder extends PrePackagedPipelineBuilderBase { - protected boolean shareUsageEnabled = true; + protected boolean shareUsageEnabled = + fiftyone.pipeline.engines.fiftyone.flowelements.Constants + .SHARE_USAGE_DEFAULT_ENABLED; private String filename; private boolean createTempDataCopy; private byte[] engineData; @@ -136,6 +140,8 @@ DeviceDetectionOnPremisePipelineBuilder setEngineData(byte[] data) { * @param enabled True to enable usage sharing. False to disable. * @return This builder instance. */ + @DefaultValue(booleanValue = fiftyone.pipeline.engines.fiftyone + .flowelements.Constants.SHARE_USAGE_DEFAULT_ENABLED) public DeviceDetectionOnPremisePipelineBuilder setShareUsage( boolean enabled) { shareUsageEnabled = enabled; @@ -150,6 +156,7 @@ public DeviceDetectionOnPremisePipelineBuilder setShareUsage( * @param enabled True to enable auto update. False to disable. * @return This builder instance. */ + @DefaultValue(booleanValue = Constants.DEFAULT_AUTOUPDATE_ENABLED) public DeviceDetectionOnPremisePipelineBuilder setAutoUpdate( boolean enabled) { autoUpdateEnabled = enabled; @@ -164,6 +171,7 @@ public DeviceDetectionOnPremisePipelineBuilder setAutoUpdate( * @param enabled True to enable file system watcher. False to disable. * @return This builder instance. */ + @DefaultValue(booleanValue = Constants.DEFAULT_WATCHER_ENABLED) public DeviceDetectionOnPremisePipelineBuilder setDataFileSystemWatcher( boolean enabled) { dataFileSystemWatcher = enabled; @@ -175,6 +183,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDataFileSystemWatcher( * @param dataUpdateService an instance of a dataUpdateService * @return This builder instance. */ + @DefaultValue("The DataUpdateService given to the constructor") public DeviceDetectionOnPremisePipelineBuilder setDataUpdateService( DataUpdateService dataUpdateService) { this.dataUpdateService = dataUpdateService; @@ -183,7 +192,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDataUpdateService( /** * Enable/Disable update on startup. - * Defaults to enabled. + * Defaults to disabled. * If enabled, the auto update system will be used to check for * an update before the device detection engine is created. * If an update is available, it will be downloaded and applied @@ -192,6 +201,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDataUpdateService( * @param enabled True to enable update on startup. False to disable. * @return This builder instance. */ + @DefaultValue(booleanValue = Constants.DEFAULT_UPDATE_ON_STARTUP) public DeviceDetectionOnPremisePipelineBuilder setDataUpdateOnStartup( boolean enabled) { dataUpdateOnStartup = enabled; @@ -205,6 +215,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDataUpdateOnStartup( * @param pollingIntervalSeconds The number of seconds between checks. * @return This builder instance. */ + @DefaultValue(intValue = Constants.DATA_UPDATE_POLLING_DEFAULT) public DeviceDetectionOnPremisePipelineBuilder setUpdatePollingInterval( int pollingIntervalSeconds) { @@ -218,6 +229,7 @@ public DeviceDetectionOnPremisePipelineBuilder setUpdatePollingInterval( * @param pollingIntervalMillis The number of milliseconds between checks. * @return This builder instance. */ + @DefaultValue(intValue = Constants.DATA_UPDATE_POLLING_DEFAULT * 1000) public DeviceDetectionOnPremisePipelineBuilder setUpdatePollingIntervalMillis( long pollingIntervalMillis) { updatePollingInterval = pollingIntervalMillis; @@ -232,6 +244,7 @@ public DeviceDetectionOnPremisePipelineBuilder setUpdatePollingIntervalMillis( * polling interval in seconds. * @return This builder instance. */ + @DefaultValue(intValue = Constants.DATA_UPDATE_RANDOMISATION_DEFAULT) public DeviceDetectionOnPremisePipelineBuilder setUpdateRandomisationMax( int randomisationMaxSeconds) { updateRandomisationMax = (long)randomisationMaxSeconds * 1000; @@ -246,6 +259,7 @@ public DeviceDetectionOnPremisePipelineBuilder setUpdateRandomisationMax( * polling interval in milliseconds. * @return This builder instance. */ + @DefaultValue(intValue = Constants.DATA_UPDATE_RANDOMISATION_DEFAULT * 1000) public DeviceDetectionOnPremisePipelineBuilder setUpdateRandomisationMaxMillis( long randomisationMaxMillis) { updateRandomisationMax = randomisationMaxMillis; @@ -259,6 +273,7 @@ public DeviceDetectionOnPremisePipelineBuilder setUpdateRandomisationMaxMillis( * @param key The license key. * @return This builder instance. */ + @DefaultValue("No license key") public DeviceDetectionOnPremisePipelineBuilder setDataUpdateLicenseKey( String key) { dataUpdateLicenseKey = key; @@ -271,6 +286,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDataUpdateLicenseKey( * @param profile The performance profile to use. * @return This builder instance. */ + @DefaultValue("Balanced") public DeviceDetectionOnPremisePipelineBuilder setPerformanceProfile( Constants.PerformanceProfiles profile) { performanceProfile = profile; @@ -284,6 +300,7 @@ public DeviceDetectionOnPremisePipelineBuilder setPerformanceProfile( * @param concurrency expected concurrent accesses * @return this builder */ + @DefaultValue("The result of Runtime#getRuntime().getAvailableProcessors()") public DeviceDetectionOnPremisePipelineBuilder setConcurrency(int concurrency) { this.concurrency = concurrency; return this; @@ -298,6 +315,7 @@ public DeviceDetectionOnPremisePipelineBuilder setConcurrency(int concurrency) { * @param difference to allow * @return this builder */ + @DefaultValue(intValue = 0) public DeviceDetectionOnPremisePipelineBuilder setDifference(int difference) { this.difference = difference; return this; @@ -314,6 +332,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDifference(int difference) { * considered valid * @return this builder */ + @DefaultValue(booleanValue = false) public DeviceDetectionOnPremisePipelineBuilder setAllowUnmatched(boolean allow) { this.allowUnmatched = allow; return this; @@ -326,6 +345,7 @@ public DeviceDetectionOnPremisePipelineBuilder setAllowUnmatched(boolean allow) * @param drift to set * @return this builder */ + @DefaultValue(intValue = 0) public DeviceDetectionOnPremisePipelineBuilder setDrift(int drift) { this.drift = drift; return this; @@ -343,6 +363,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDrift(int drift) { * @param use true if the performance graph should be used * @return this builder */ + @DefaultValue(booleanValue = false) public DeviceDetectionOnPremisePipelineBuilder setUsePerformanceGraph(boolean use) { this.usePerformanceGraph = use; return this; @@ -359,6 +380,7 @@ public DeviceDetectionOnPremisePipelineBuilder setUsePerformanceGraph(boolean us * @param use true if the predictive graph should be used * @return this builder */ + @DefaultValue(booleanValue = true) public DeviceDetectionOnPremisePipelineBuilder setUsePredictiveGraph(boolean use) { this.usePredictiveGraph = use; return this; @@ -370,6 +392,7 @@ public DeviceDetectionOnPremisePipelineBuilder setUsePredictiveGraph(boolean use * @param property the property that we want the engine to populate * @return this builder */ + @DefaultValue("All properties") public DeviceDetectionOnPremisePipelineBuilder setProperty(String property) { this.properties.add(property); return this; @@ -384,6 +407,7 @@ public DeviceDetectionOnPremisePipelineBuilder setProperty(String property) { * @param url the URL to check for a new data file * @return this builder */ + @DefaultValue("51Degrees default, set by the engine's builder") public DeviceDetectionOnPremisePipelineBuilder setDataUpdateUrl(String url) { dataUpdateUrl = url; return this; @@ -400,6 +424,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDataUpdateUrl(String url) { * False otherwise * @return this builder */ + @DefaultValue(booleanValue = Constants.DEFAULT_VERIFY_MD5) public DeviceDetectionOnPremisePipelineBuilder setDataUpdateVerifyMd5(Boolean verify) { dataUpdateVerifyMd5 = verify; return this; @@ -413,6 +438,7 @@ public DeviceDetectionOnPremisePipelineBuilder setDataUpdateVerifyMd5(Boolean ve * @param formatter the formatter to use * @return this builder */ + @DefaultValue("51Degrees default, set by the engine's builder") public DeviceDetectionOnPremisePipelineBuilder setDataUpdateUrlFormatter(DataUpdateUrlFormatter formatter) { dataUpdateUrlFormatter = formatter; return this; @@ -431,22 +457,33 @@ public Pipeline build() throws Exception { new DeviceDetectionHashEngineBuilder(loggerFactory, dataUpdateService); deviceDetectionEngine = configureAndBuild(hashBuilder); + List> made = new ArrayList<>(); if (deviceDetectionEngine != null) { - // Add the share usage element to the list if enabled - if (shareUsageEnabled) { - getFlowElements().add( - new ShareUsageBuilder(loggerFactory).build()); - } // Add the device detection engine to the list + made.add(deviceDetectionEngine); getFlowElements().add(deviceDetectionEngine); + // Usage sharing goes last, so that it shares what every other + // Flow Element has added to the Evidence. + if (shareUsageEnabled) { + FlowElement shareUsage = + new ShareUsageBuilder(loggerFactory).build(); + made.add(shareUsage); + getFlowElements().add(shareUsage); + } } else { throw new RuntimeException("Unexpected error creating device detection engine."); } setAutoCloseElements(true); - // Create and return the pipeline - return super.build(); + // Create and return the pipeline, closing what this builder made if + // the Pipeline is refused. + try { + return super.build(); + } catch (Exception e) { + closeMadeElements(made, e); + throw e; + } } /** diff --git a/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionPipelineBuilder.java b/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionPipelineBuilder.java index 8985dfd27..6e025c28b 100644 --- a/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionPipelineBuilder.java +++ b/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionPipelineBuilder.java @@ -145,7 +145,7 @@ public DeviceDetectionOnPremisePipelineBuilder useOnPremise( /** * Use the 51Degrees Cloud service to perform device detection. * @param resourceKey The resource key to use when querying the cloud service. - * Obtain one from https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=device-detection-src-main-java-fiftyone-devicedetection-devicedetectionpipelinebuilder.java&utm_term=usecloud + * Obtain one from https://51degrees.com/configure?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=device-detection-src-main-java-fiftyone-devicedetection-devicedetectionpipelinebuilder.java&utm_term=usecloud * @return A builder that can be used to configure and build a pipeline * that will use the cloud device detection engine. */ diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/BuilderDefaultValueTests.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/BuilderDefaultValueTests.java new file mode 100644 index 000000000..3d04cdbfb --- /dev/null +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/BuilderDefaultValueTests.java @@ -0,0 +1,70 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection; + +import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngineBuilder; +import fiftyone.pipeline.annotations.DefaultValue; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Every configuration setter of the Device Detection on-premise builders + * gives its default value beside it, with {@link DefaultValue}. See + * reference-implementation-notes.md line 79: + * "In all cases, there SHOULD be some mechanism to specify the default + * values alongside configuration property setters." + */ +public class BuilderDefaultValueTests { + + static List missingDefaults(Class... builders) { + List missing = new ArrayList<>(); + for (Class builder : builders) { + for (Method method : builder.getDeclaredMethods()) { + if (Modifier.isPublic(method.getModifiers()) && + method.getName().startsWith("set") && + method.isSynthetic() == false && + method.isBridge() == false && + method.getAnnotation(DefaultValue.class) == null) { + missing.add(builder.getSimpleName() + "#" + + method.getName() + + Arrays.toString(method.getParameterTypes())); + } + } + } + return missing; + } + + @Test + public void builderSetters_GiveTheirDefaults() { + assertEquals(new ArrayList(), missingDefaults( + DeviceDetectionOnPremisePipelineBuilder.class, + DeviceDetectionHashEngineBuilder.class)); + } +} diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/CloudOnPremiseSubstitutionTests.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/CloudOnPremiseSubstitutionTests.java new file mode 100644 index 000000000..e2c3b95f3 --- /dev/null +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/CloudOnPremiseSubstitutionTests.java @@ -0,0 +1,101 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection; + +import fiftyone.devicedetection.cloud.flowelements.DeviceDetectionCloudEngine; +import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; +import fiftyone.devicedetection.shared.DeviceData; +import fiftyone.pipeline.core.flowelements.FlowElement; +import org.junit.Test; + +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; + +/** + * The cloud and on-premise Device Detection engines put their results in + * Flow Data under the same key and as the same interface, so a consuming + * application reads them in the same way whichever engine is in the + * Pipeline. See + * README.md line 95: + * "Where both Cloud and On-Premise variants of an Engine are available, + * they MUST arrange that the Properties and values produced are compatible + * with each other so that the Engines can be substituted in the Pipeline + * without alteration to the consuming application." + *

+ * Comparing the values each engine gives needs a resource key and the + * native engine with a data file, so it is left to the example tests. + */ +public class CloudOnPremiseSubstitutionTests { + + /** + * The engines are created without running a constructor, as only the + * data key, which is a constant, is read from them. + */ + @Test + public void elementDataKey_IsTheSame() { + FlowElement cloud = + mock(DeviceDetectionCloudEngine.class, CALLS_REAL_METHODS); + FlowElement onPremise = + mock(DeviceDetectionHashEngine.class, CALLS_REAL_METHODS); + assertEquals( + onPremise.getElementDataKey(), + cloud.getElementDataKey()); + } + + /** + * Each engine's Element Data is a {@link DeviceData}, which the + * consuming application asks Flow Data for. + */ + @Test + public void elementData_IsDeviceData() { + assertTrue(DeviceData.class.isAssignableFrom( + elementDataType(DeviceDetectionCloudEngine.class))); + assertTrue(DeviceData.class.isAssignableFrom( + elementDataType(DeviceDetectionHashEngine.class))); + } + + /** + * The first type argument of the nearest generic superclass, which is + * the engine's Element Data type. + */ + private static Class elementDataType(Class engine) { + for (Class c = engine; c != null; c = c.getSuperclass()) { + Type type = c.getGenericSuperclass(); + if (type instanceof ParameterizedType) { + Type first = + ((ParameterizedType) type).getActualTypeArguments()[0]; + if (first instanceof Class) { + return (Class) first; + } + } + } + assertNotNull("No Element Data type found for " + engine, null); + return null; + } +} diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionCloudPipelineBuilderTests.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionCloudPipelineBuilderTests.java new file mode 100644 index 000000000..2d9e0cbd3 --- /dev/null +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionCloudPipelineBuilderTests.java @@ -0,0 +1,252 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection; + +import fiftyone.devicedetection.shared.DeviceData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.services.HttpClient; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * The cloud Pipeline builder passes the keys and the requested properties + * to the Cloud Request Engine, so a Pipeline can authenticate on a license + * key alone. A fake cloud service records each data request. + */ +public class DeviceDetectionCloudPipelineBuilderTests { + + private static final String BASE = "https://cloud.example/api/v4/"; + private static final String LICENSE_KEY = "abcdefghijkl1234"; + private static final String RESOURCE_KEY = "resourcekey"; + private static final String FIRST = "device.ismobile"; + private static final String SECOND = "device.iscrawler"; + private static final String PROPERTIES_RESPONSE = + "{\"Products\":{\"device\":{\"DataTier\":\"t\",\"Properties\":" + + "[{\"Name\":\"IsMobile\",\"Type\":\"Boolean\"," + + "\"Category\":\"c\"}]}}}"; + private static final String DATA_RESPONSE = "{\"device\":{\"ismobile\":true}}"; + + /** Records every data request and answers from fixed responses. */ + private static class FakeCloud implements HttpClient { + final List dataRequests = + Collections.synchronizedList(new ArrayList()); + + @Override + public HttpURLConnection connect(URL url) { + return new HttpURLConnection(url) { + @Override + public void connect() { + } + + @Override + public void disconnect() { + } + + @Override + public boolean usingProxy() { + return false; + } + + @Override + public int getResponseCode() { + return 200; + } + }; + } + + @Override + public String postData( + HttpURLConnection connection, + Map headers, + byte[] data) { + return answer( + connection.getURL(), + data == null ? "" : + new String(data, StandardCharsets.UTF_8)); + } + + @Override + public String getResponseString(HttpURLConnection connection) { + return answer(connection.getURL(), ""); + } + + @Override + public String getResponseString( + HttpURLConnection connection, + Map headers) { + return answer(connection.getURL(), ""); + } + + private String answer(URL url, String body) { + String path = url.getPath(); + if (path.endsWith("accessibleproperties")) { + return PROPERTIES_RESPONSE; + } + if (path.endsWith("evidencekeys")) { + return "[\"header.user-agent\"]"; + } + dataRequests.add(body); + return DATA_RESPONSE; + } + } + + private FakeCloud cloud; + + @Before + public void setUp() { + cloud = new FakeCloud(); + } + + private DeviceDetectionCloudPipelineBuilder builder(String resourceKey) { + return new DeviceDetectionPipelineBuilder( + LoggerFactory.getILoggerFactory(), cloud) + .useCloud(resourceKey) + .setEndPoint(BASE); + } + + /** Process one request and give the fields of the form it sent. */ + private Map process(DeviceDetectionCloudPipelineBuilder builder) + throws Exception { + try (Pipeline pipeline = builder.build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "iPhone").process(); + assertTrue(data.get(DeviceData.class) + .getIsMobile().getValue()); + } + assertEquals(1, cloud.dataRequests.size()); + return form(cloud.dataRequests.get(0)); + } + + private static Map form(String body) + throws UnsupportedEncodingException { + Map fields = new HashMap<>(); + for (String pair : body.split("&")) { + if (pair.isEmpty()) { + continue; + } + String[] parts = pair.split("=", 2); + fields.put( + URLDecoder.decode(parts[0], "UTF-8"), + URLDecoder.decode(parts.length > 1 ? parts[1] : "", "UTF-8")); + } + return fields; + } + + @Test + public void LicenseKeyAlone_RequestedProperties_SentAsValues() + throws Exception { + Map sent = process(builder(null) + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Arrays.asList(FIRST, SECOND))); + + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST + "," + SECOND, sent.get("values")); + assertFalse(sent.containsKey("resource")); + } + + @Test + public void LicenseKeyAlone_RequestedPropertiesAsText_SentAsValues() + throws Exception { + Map sent = process(builder("") + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(FIRST + ", " + SECOND)); + + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST + "," + SECOND, sent.get("values")); + assertFalse(sent.containsKey("resource")); + } + + @Test + public void BothKeys_RequestedProperties_SendsAllThree() + throws Exception { + Map sent = process(builder(RESOURCE_KEY) + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Collections.singletonList(FIRST))); + + assertEquals(RESOURCE_KEY, sent.get("resource")); + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST, sent.get("values")); + } + + @Test + public void ResourceKey_NoRequestedProperties_SendsNoValues() + throws Exception { + Map sent = process(builder(RESOURCE_KEY)); + + assertEquals(RESOURCE_KEY, sent.get("resource")); + assertFalse(sent.containsKey("license")); + assertFalse(sent.containsKey("values")); + } + + @Test + public void LicenseKeyAlone_NoRequestedProperties_Refused() + throws Exception { + try { + builder(null).setLicenseKey(LICENSE_KEY).build().close(); + fail("A license key with no requested properties was built."); + } catch (PipelineConfigurationException e) { + assertTrue(e.getMessage(), + e.getMessage().contains("answers 400")); + assertTrue(e.getMessage(), + e.getMessage().contains("setRequestedProperties")); + assertFalse(e.getMessage().contains(LICENSE_KEY)); + } + assertTrue(cloud.dataRequests.isEmpty()); + } + + @Test + public void SetRequestedProperties_Null_Refused() { + try { + builder(RESOURCE_KEY).setRequestedProperties((List) null); + fail("A null list was accepted."); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("properties")); + } + try { + builder(RESOURCE_KEY).setRequestedProperties((String) null); + fail("A null string was accepted."); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("properties")); + } + } +} diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderDefaultsTest.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderDefaultsTest.java new file mode 100644 index 000000000..cec79e1ff --- /dev/null +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderDefaultsTest.java @@ -0,0 +1,49 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection; + +import fiftyone.pipeline.engines.fiftyone.flowelements.Constants; +import fiftyone.pipeline.engines.services.DataUpdateService; +import fiftyone.pipeline.engines.services.HttpClient; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +/** + * Usage sharing is on unless the builder is told otherwise, as + * {@link Constants#SHARE_USAGE_DEFAULT_ENABLED} says. + */ +public class DeviceDetectionOnPremisePipelineBuilderDefaultsTest { + + @Test + public void ShareUsage_NotSet_FollowsDefault() { + DeviceDetectionOnPremisePipelineBuilder builder = + new DeviceDetectionOnPremisePipelineBuilder( + mock(DataUpdateService.class), + mock(HttpClient.class)); + assertEquals( + Constants.SHARE_USAGE_DEFAULT_ENABLED, + builder.shareUsageEnabled); + } +} diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java index aed639665..75f38b64d 100644 --- a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java @@ -24,13 +24,14 @@ import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.ConfigHashSwig; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; +import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.configuration.DataFileConfiguration; import fiftyone.pipeline.engines.data.AspectEngineDataFile; import fiftyone.pipeline.engines.data.DataUpdateUrlFormatter; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneDataFileDefault; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; import fiftyone.pipeline.engines.fiftyone.flowelements.ShareUsageElement; import fiftyone.pipeline.engines.services.DataUpdateServiceDefault; import fiftyone.pipeline.util.FileFinder; @@ -51,7 +52,7 @@ @SuppressWarnings({"rawtypes", "deprecation"}) public class DeviceDetectionOnPremisePipelineBuilderTest { - private static final String HASH_DATA_FILE_NAME = FileUtils.getHashFileName(); + private static final String HASH_DATA_FILE_NAME = TestFileUtils.getHashFileName(); private DeviceDetectionOnPremisePipelineBuilder builder; private File datafile = FileFinder.getFilePath(HASH_DATA_FILE_NAME); private ILoggerFactory loggerFactory; @@ -164,7 +165,30 @@ public void setShareUsage_True_SharedFlowElementAdded() throws Exception { .build()) { List flowElements = pipeline.getFlowElements(); assertEquals(2, flowElements.size()); - assertTrue(flowElements.get(0) instanceof ShareUsageElement); + // Usage sharing is the last Flow Element. + assertTrue(flowElements.get(flowElements.size() - 1) + instanceof ShareUsageElement); + } + } + + /** + * An element the caller adds after turning usage sharing on still goes + * before usage sharing, which the builder adds when it builds. + */ + @Test + public void setShareUsage_True_ElementAddedAfter_ShareUsageLast() + throws Exception { + try (Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) + .useOnPremise(datafile.getAbsolutePath(), false) + .setAutoUpdate(false) + .setShareUsage(true) + .addFlowElement(new SequenceElementBuilder(loggerFactory).build()) + .build()) { + List flowElements = pipeline.getFlowElements(); + assertEquals(3, flowElements.size()); + assertTrue(flowElements.toString(), + flowElements.get(flowElements.size() - 1) + instanceof ShareUsageElement); } } diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionTests.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionTests.java index a59a0a6d6..8a9fe3159 100644 --- a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionTests.java +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionTests.java @@ -25,7 +25,7 @@ import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.shared.DeviceData; import fiftyone.devicedetection.shared.testhelpers.UserAgentGenerator; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; +import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.data.FlowError; import fiftyone.pipeline.core.flowelements.Pipeline; @@ -52,7 +52,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.UA_FILE_PATH; +import static fiftyone.devicedetection.shared.testhelpers.TestFileUtils.UA_FILE_PATH; import static fiftyone.pipeline.core.Constants.EVIDENCE_HTTPHEADER_PREFIX; import static fiftyone.pipeline.core.Constants.EVIDENCE_SEPERATOR; import static fiftyone.pipeline.engines.Constants.PerformanceProfiles.*; @@ -68,7 +68,7 @@ public class DeviceDetectionTests { private static final Logger logger = LoggerFactory.getLogger(DeviceDetectionTests.class); - private static final String HASH_DATA_FILE_NAME = FileUtils.getHashFileName(); + private static final String HASH_DATA_FILE_NAME = TestFileUtils.getHashFileName(); private static UserAgentGenerator userAgents; TestConfig[] hashConfigs = { diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/PublishedPackagesHoldNoTestsTests.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/PublishedPackagesHoldNoTestsTests.java new file mode 100644 index 000000000..9bea4818b --- /dev/null +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/PublishedPackagesHoldNoTestsTests.java @@ -0,0 +1,122 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection; + +import fiftyone.devicedetection.cloud.flowelements.DeviceDetectionCloudEngine; +import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; +import fiftyone.devicedetection.shared.DeviceDataBase; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertTrue; + +/** + * Examples and tests SHOULD NOT be included in any published packages + * (device-detection-specification/packaging-and-structure.md line 16 at + * 706de1f0). Lists what each published Device Detection package's main jar + * holds, being the classes folder in a reactor build or the jar otherwise, + * and looks for test helpers, tests and examples. + */ +public class PublishedPackagesHoldNoTestsTests { + + /** + * A main class of each published Device Detection package. + */ + private static final Class[] PACKAGES = { + DeviceDataBase.class, + DeviceDetectionHashEngine.class, + DeviceDetectionCloudEngine.class, + DeviceDetectionPipelineBuilder.class + }; + + private static List entriesOf(Class type) + throws IOException, URISyntaxException { + File location = new File(type.getProtectionDomain() + .getCodeSource().getLocation().toURI()); + List entries = new ArrayList<>(); + if (location.isDirectory()) { + Path root = location.toPath(); + try (Stream paths = Files.walk(root)) { + entries.addAll(paths + .map(p -> root.relativize(p).toString().replace('\\', '/')) + .collect(Collectors.toList())); + } + } else { + try (JarFile jar = new JarFile(location)) { + Enumeration e = jar.entries(); + while (e.hasMoreElements()) { + entries.add(e.nextElement().getName()); + } + } + } + return entries; + } + + /** + * Test helpers that stay in the published jar, deprecated, because a + * patch release removes no public name, and the folder a jar lists for + * them. A later release removes them. + */ + private static final List KEPT_DEPRECATED = Arrays.asList( + "fiftyone/devicedetection/shared/testhelpers/", + "fiftyone/devicedetection/shared/testhelpers/FileUtils.class", + "fiftyone/devicedetection/shared/testhelpers/KeyUtils.class"); + + private static boolean isTestOrExample(String entry) { + if (KEPT_DEPRECATED.contains(entry)) { + return false; + } + String lower = entry.toLowerCase(); + return lower.contains("testhelpers/") || + lower.contains("/examples/") || + lower.endsWith("test.class") || + lower.endsWith("tests.class"); + } + + @Test + public void PublishedPackages_HoldNoTestsOrExamples() throws Exception { + List found = new ArrayList<>(); + for (Class type : PACKAGES) { + for (String entry : entriesOf(type)) { + if (isTestOrExample(entry)) { + found.add(type.getSimpleName() + "'s package: " + entry); + } + } + } + assertTrue("Published packages hold tests or examples: " + found, + found.isEmpty()); + } +} diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/RefusedBuildTests.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/RefusedBuildTests.java new file mode 100644 index 000000000..88212f9b7 --- /dev/null +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/RefusedBuildTests.java @@ -0,0 +1,157 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection; + +import fiftyone.devicedetection.shared.testhelpers.TestFileUtils; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.engines.fiftyone.flowelements.ShareUsageBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.ShareUsageElement; +import fiftyone.pipeline.engines.services.DataUpdateServiceDefault; +import fiftyone.pipeline.engines.services.HttpClient; +import fiftyone.pipeline.engines.services.HttpClientDefault; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.AbstractList; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.junit.Assume.assumeTrue; + +/** + * When the on-premise builder's Pipeline is refused, the builder closes the + * Flow Elements it made itself, because no caller holds them to close, and + * leaves the caller's own Flow Elements open. + * @see Specification + */ +@SuppressWarnings("rawtypes") +public class RefusedBuildTests { + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + /** + * Records every Flow Element the builder adds to its own list, which are + * the ones it made. + */ + private static class RecordingBuilder + extends DeviceDetectionOnPremisePipelineBuilder { + private final List made = new ArrayList<>(); + + RecordingBuilder(ILoggerFactory loggerFactory, HttpClient httpClient) { + super( + loggerFactory, + new DataUpdateServiceDefault( + loggerFactory.getLogger( + DataUpdateServiceDefault.class.getName()), + httpClient), + httpClient); + } + + List elements() { + return super.getFlowElements(); + } + + @Override + protected List getFlowElements() { + final List elements = super.getFlowElements(); + return new AbstractList() { + @Override + public FlowElement get(int index) { + return elements.get(index); + } + + @Override + public int size() { + return elements.size(); + } + + @Override + public void add(int index, FlowElement element) { + made.add(element); + elements.add(index, element); + } + + @Override + public FlowElement set(int index, FlowElement element) { + return elements.set(index, element); + } + + @Override + public FlowElement remove(int index) { + return elements.remove(index); + } + }; + } + } + + private static File dataFile() { + String name = TestFileUtils.getHashFileName(); + assumeTrue("No Hash data file was found", name != null); + return new File(name); + } + + /** + * Usage sharing the caller adds goes before the engine and the usage + * sharing element the builder adds, so the Pipeline is refused. + */ + @Test + public void Refused_ElementsTheBuilderMadeAreClosed() throws Exception { + RecordingBuilder builder = + new RecordingBuilder(loggerFactory, new HttpClientDefault()); + builder.setFilename(dataFile().getAbsolutePath(), false) + .setAutoUpdate(false) + .setDataFileSystemWatcher(false); + ShareUsageElement callers = + new ShareUsageBuilder(loggerFactory).build(); + try { + builder.addFlowElement(callers); + try { + builder.build().close(); + fail("A Pipeline with usage sharing first was built"); + } catch (PipelineConfigurationException e) { + assertTrue(e.getMessage(), + e.getMessage().contains("must be the last Flow Element")); + } + + assertFalse("The builder made no Flow Elements", + builder.made.isEmpty()); + for (FlowElement element : builder.made) { + assertTrue(element + " that the builder made is open", + element.isClosed()); + assertFalse(element + " is still in the builder's list", + builder.elements().contains(element)); + } + assertFalse("The caller's own Flow Element was closed", + callers.isClosed()); + } finally { + callers.close(); + } + } +} diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/data/EnginesInterfaceCompatibleTests.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/data/EnginesInterfaceCompatibleTests.java new file mode 100644 index 000000000..1564f894a --- /dev/null +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/data/EnginesInterfaceCompatibleTests.java @@ -0,0 +1,93 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.data; + +import fiftyone.devicedetection.cloud.data.DeviceDataCloud; +import fiftyone.devicedetection.cloud.data.MultiDeviceDataCloud; +import fiftyone.devicedetection.hash.engine.onpremise.data.DeviceDataHash; +import fiftyone.devicedetection.shared.DeviceData; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Each of these Engines MUST return data that is interface compatible with + * the others (device-detection-specification/data-model.md line 15 at + * 706de1f0). The on-premise and cloud data share the DeviceData interface, + * so code written against it reads either, and the cloud hardware profile + * lookup gives its profiles as DeviceData too. + */ +public class EnginesInterfaceCompatibleTests { + + @Test + public void Data_OnPremiseAndCloudAreDeviceData() { + assertTrue("On-premise data is not DeviceData", + DeviceData.class.isAssignableFrom(DeviceDataHash.class)); + assertTrue("Cloud data is not DeviceData", + DeviceData.class.isAssignableFrom(DeviceDataCloud.class)); + } + + /** + * Every property accessor of DeviceData is reached the same way, with the + * same type, through the on-premise and the cloud data. + */ + @Test + public void Data_SameAccessorsAndTypes() throws Exception { + int accessors = 0; + for (Method method : DeviceData.class.getMethods()) { + if (method.getDeclaringClass().equals(DeviceData.class) && + AspectPropertyValue.class.equals(method.getReturnType())) { + for (Class data : new Class[] { + DeviceDataHash.class, DeviceDataCloud.class}) { + Method other = data.getMethod( + method.getName(), method.getParameterTypes()); + assertEquals(data.getSimpleName() + "." + method.getName(), + method.getGenericReturnType(), + other.getGenericReturnType()); + } + accessors++; + } + } + assertTrue("DeviceData has no property accessors", accessors > 0); + } + + /** + * The hardware profile lookup gives a list of DeviceData. + */ + @Test + public void Data_HardwareProfilesAreDeviceData() throws Exception { + Type type = MultiDeviceDataCloud.class.getMethod("getProfiles") + .getGenericReturnType(); + assertTrue(type instanceof ParameterizedType); + ParameterizedType list = (ParameterizedType) type; + assertEquals(List.class, list.getRawType()); + assertEquals(DeviceData.class, list.getActualTypeArguments()[0]); + } +} diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/data/EnginesPopulateAspectDataTests.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/data/EnginesPopulateAspectDataTests.java new file mode 100644 index 000000000..03705d692 --- /dev/null +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/data/EnginesPopulateAspectDataTests.java @@ -0,0 +1,103 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.data; + +import fiftyone.devicedetection.cloud.data.MultiDeviceDataCloud; +import fiftyone.devicedetection.cloud.flowelements.DeviceDataCloudInternal; +import fiftyone.devicedetection.cloud.flowelements.DeviceDetectionCloudEngine; +import fiftyone.devicedetection.cloud.flowelements.HardwareProfileCloudEngine; +import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDataHashDefault; +import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectDataBase; +import fiftyone.pipeline.engines.flowelements.AspectEngine; +import org.junit.Test; + +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertTrue; + +/** + * All Engines MUST populate data objects that implement Aspect Data as + * defined in the Pipeline specification + * (device-detection-specification/data-model.md line 10 at 706de1f0). + * Each Device Detection engine is checked through the data type it is + * declared with, and each class that is made to hold an engine's data. + */ +public class EnginesPopulateAspectDataTests { + + private static final List> ENGINES = Arrays.asList( + DeviceDetectionHashEngine.class, + DeviceDetectionCloudEngine.class, + HardwareProfileCloudEngine.class); + + /** + * Find the data type an engine is declared with, which is the first type + * argument of the base class it extends. + */ + private static Class dataTypeOf(Class engine) { + Type type = engine.getGenericSuperclass(); + assertTrue(engine.getSimpleName() + " has no data type argument", + type instanceof ParameterizedType); + Type data = ((ParameterizedType) type).getActualTypeArguments()[0]; + assertTrue(engine.getSimpleName() + "'s data type is not a class", + data instanceof Class); + return (Class) data; + } + + @Test + public void Engines_AreAspectEngines() { + for (Class engine : ENGINES) { + assertTrue(engine.getSimpleName() + " is not an Aspect Engine", + AspectEngine.class.isAssignableFrom(engine)); + } + } + + @Test + public void Engines_DeclaredDataIsAspectData() { + for (Class engine : ENGINES) { + Class data = dataTypeOf(engine); + assertTrue(engine.getSimpleName() + " populates " + + data.getSimpleName() + ", which is not Aspect Data", + AspectData.class.isAssignableFrom(data)); + } + } + + /** + * The classes the engines' factories make, which hold the values, take + * the Pipeline's Aspect Data behaviour from its base class. + */ + @Test + public void Engines_DataMadeIsAspectData() { + for (Class made : Arrays.>asList( + DeviceDataHashDefault.class, + DeviceDataCloudInternal.class, + MultiDeviceDataCloud.class)) { + assertTrue(made.getSimpleName() + " is not Aspect Data", + AspectDataBase.class.isAssignableFrom(made)); + } + } +} diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/data/StoredValueAsStringTests.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/data/StoredValueAsStringTests.java new file mode 100644 index 000000000..b6cd95da4 --- /dev/null +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/data/StoredValueAsStringTests.java @@ -0,0 +1,227 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.data; + +import fiftyone.devicedetection.shared.DeviceDataBaseOnPremise; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.ElementPropertyMetaDataDefault; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.types.JavaScript; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; +import fiftyone.pipeline.engines.data.ProvidesValuesAsString; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.flowelements.AspectEngine; +import fiftyone.pipeline.engines.services.MissingPropertyReason; +import fiftyone.pipeline.engines.services.MissingPropertyResult; +import fiftyone.pipeline.engines.services.MissingPropertyService; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * On-premise Device Data holds Property values as strings, and the string + * stored for a Property can be read as well as the parsed value. + *

+ * Specification: pipeline-specification/features/access-to-results.md lines + * 65 to 67 at 51Degrees/specifications 706de1f0, "Where an Element Data + * instance holds Property values as strings and parses them into the type + * each Property declares, it SHOULD be possible to get the stored string for + * a Property as well as the parsed value.", and value-types.md lines 39 to + * 53 for what an implementation offering it must do. + */ +public class StoredValueAsStringTests { + + private static final String MISSING_DESCRIPTION = "Not in this file."; + + private TestData data; + + @Before + @SuppressWarnings("unchecked") + public void init() { + AspectEngine + engine = mock(AspectEngine.class); + when(engine.getElementDataKey()).thenReturn("device"); + Map properties = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + properties.put("ismobile", new ElementPropertyMetaDataDefault( + "ismobile", engine, "category", Boolean.class, true)); + properties.put("screenpixelswidth", new ElementPropertyMetaDataDefault( + "screenpixelswidth", engine, "category", Integer.class, true)); + Map> available = + new HashMap<>(); + available.put("device", properties); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElementAvailableProperties()).thenReturn(available); + FlowData flowData = mock(FlowData.class); + when(flowData.getPipeline()).thenReturn(pipeline); + MissingPropertyService missing = mock(MissingPropertyService.class); + when(missing.getMissingPropertyReason(anyString(), any(List.class))) + .thenReturn(new MissingPropertyResult( + MissingPropertyReason.DataFileUpgradeRequired, + MISSING_DESCRIPTION)); + + Map stored = new HashMap<>(); + stored.put("ismobile", "Unknown"); + stored.put("screenpixelswidth", "N/A"); + data = new TestData(flowData, engine, missing, stored); + } + + @Test + public void capability_CanBeTestedFor() { + assertTrue(data instanceof ProvidesValuesAsString); + } + + @Test + public void storedString_IsReturnedAsStored() { + ProvidesValuesAsString strings = data; + + assertEquals("Unknown", + strings.getStoredValueAsString("ismobile").getValue()); + assertEquals("N/A", + strings.getStoredValueAsString("ScreenPixelsWidth").getValue()); + } + + @Test + public void typedAccessor_IsUnchanged() { + data.getStoredValueAsString("ismobile"); + + assertFalse(data.getIsMobile().getValue()); + assertEquals(0, (int) data.getScreenPixelsWidth().getValue()); + } + + @Test + public void missingProperty_ReportedAsByTheTypedAccessor() { + String typed = null; + try { + data.getIsTablet(); + fail("The typed accessor throws for a missing property"); + } catch (PropertyMissingException e) { + typed = e.getMessage(); + } + try { + data.getStoredValueAsString("istablet"); + fail("The stored value throws for a missing property"); + } catch (PropertyMissingException e) { + assertEquals(typed, e.getMessage()); + assertEquals(MissingPropertyReason.DataFileUpgradeRequired, + e.getReason()); + } + } + + /** + * Device Data that holds its values as strings and parses them as the C + * library does. + */ + private static class TestData extends DeviceDataBaseOnPremise { + private final Map stored; + + TestData( + FlowData flowData, + AspectEngine engine, + MissingPropertyService missingPropertyService, + Map stored) { + super(mock(Logger.class), flowData, engine, + missingPropertyService); + this.stored = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + this.stored.putAll(stored); + } + + @Override + protected boolean propertyIsAvailable(String propertyName) { + return stored.containsKey(propertyName); + } + + @Override + public AspectPropertyValue> getValues( + String propertyName) { + throw new UnsupportedOperationException(); + } + + @Override + protected AspectPropertyValue getValueAsString( + String propertyName) { + return new AspectPropertyValueDefault<>(stored.get(propertyName)); + } + + @Override + protected AspectPropertyValue getValueAsInteger( + String propertyName) { + String value = stored.get(propertyName); + int parsed = 0; + try { + parsed = Integer.parseInt(value); + } catch (NumberFormatException e) { + // As C's atoi, text that is not a number gives 0. + } + return new AspectPropertyValueDefault<>(parsed); + } + + @Override + protected AspectPropertyValue getValueAsBool( + String propertyName) { + return new AspectPropertyValueDefault<>( + "True".equals(stored.get(propertyName))); + } + + @Override + protected AspectPropertyValue getValueAsDouble( + String propertyName) { + throw new UnsupportedOperationException(); + } + + @Override + protected AspectPropertyValue getValueAsJavaScript( + String propertyName) { + throw new UnsupportedOperationException(); + } + + @Override + public AspectPropertyValue> getUserAgents() { + throw new UnsupportedOperationException(); + } + + @Override + public AspectPropertyValue getDeviceId() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/device-detection/pom.xml b/device-detection/pom.xml index b255d1e53..cea9ab984 100644 --- a/device-detection/pom.xml +++ b/device-detection/pom.xml @@ -56,7 +56,7 @@ 3.7.1 6.5.3 0.7.0 - 3.26.0 + 3.28.0 true false @@ -177,13 +177,12 @@ ${maven.multiModuleProjectDirectory} - - ${TestResourceKey} - - ${SuperResourceKey} - - ${LicenseKey} + + <_51DEGREES_RESOURCE_KEY>${_51DEGREES_RESOURCE_KEY} + <_51DEGREES_RESOURCE_KEY_SUPER>${_51DEGREES_RESOURCE_KEY_SUPER} + <_51DEGREES_LICENSE_KEY>${_51DEGREES_LICENSE_KEY} @@ -363,9 +362,11 @@ - - https://github.com/51Degrees/device-detection-java - scm:git:git://github.com/51Degrees/device-detection-java.git - scm:git:info@51degrees.com:51Degrees/device-detection-java.git + + https://github.com/51Degrees/51degrees-java + scm:git:https://github.com/51Degrees/51degrees-java.git + scm:git:https://github.com/51Degrees/51degrees-java.git diff --git a/examples/device-detection/README.md b/examples/device-detection/README.md index c83d69956..448cc12d2 100644 --- a/examples/device-detection/README.md +++ b/examples/device-detection/README.md @@ -40,15 +40,16 @@ argument to use it. You will require [resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys) to use the Cloud API, as described on our website. Get resource keys from -our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys) on +our [configuration page](https://51degrees.com/configure/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys) on how to use this. A resource key configured with the properties needed -to run most of the examples can be obtained [here](https://configure.51degrees.com/jqz435Nc?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys). +to run most of the examples can be obtained [here](https://51degrees.com/configure/jqz435Nc?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-resource-keys). To use the resource key in the example it can be supplied as a command line parameter, pasted into the configuration file (where there is one) or supplied as either an environment variable or a system -property called "TestResourceKey". +property called "_51DEGREES_RESOURCE_KEY". The TAC and native model +examples read theirs from "_51DEGREES_RESOURCE_KEY_SUPER". Some cloud examples require an enhanced resource key containing a license key. And some on-premise examples require you to provide a license key. You can find out about @@ -94,6 +95,7 @@ The tables below describe the examples available in this repository. | OfflineProcessing | Example showing how to ingest a file containing data from web requests and perform detection against the entries. | | PerformanceBenchmark | How to configure the various performance options and run some simple performance tests. | | UpdateOnStartup | How to configure the Pipeline to automatically update the device detection data file on startup. Also illustrates 'file watcher'. This will refresh the device detection engine if the specified data file is updated on disk. | +| ManualDataUpdate | How to make the device detection engine load a new data file when the application decides to, rather than through an automatic update, with no restart. | ## Running built examples from command line @@ -103,23 +105,24 @@ Running mvn package ``` -will produce "fat" JARs inside `target` subfolders. +will produce "fat" JARs inside `target` subfolders. The version in each name is the one in this folder's `pom.xml`, which is not +the version of the packages the examples use. Use them with relevant example class entrypoints like: ```bash -java -cp ./console/target/device-detection-java-examples.console-4.4.20-jar-with-dependencies.jar fiftyone.devicedetection.examples.console.OfflineProcessing +java -cp ./console/target/device-detection-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar fiftyone.devicedetection.examples.console.OfflineProcessing ``` ### Web examples The web examples are run from the root of this repository, because they find their -web content by a path relative to it. They listen on port 8081 and stop when Enter +web content by a path relative to it. They listen on port 5151 and stop when Enter is pressed. Set the `PORT` environment variable to listen on another port, in which case the example runs until the process is stopped: ```bash -PORT=8100 java -jar ./web/getting-started.onprem/target/device-detection-java-examples.web.getting-started.onprem-4.4.20-jar-with-dependencies.jar +PORT=8100 java -jar ./web/getting-started.onprem/target/device-detection-java-examples.web.getting-started.onprem-4.5.127-SNAPSHOT-jar-with-dependencies.jar ``` The on-premise web example takes its data file from the `_51DEGREES_DD_PATH` @@ -137,7 +140,7 @@ The fat JARs above bundle everything into a single jar on the classpath, so the permission to use is `ALL-UNNAMED`: ```bash -java -cp ./console/target/device-detection-java-examples.console-4.4.20-jar-with-dependencies.jar --enable-native-access=ALL-UNNAMED fiftyone.devicedetection.examples.console.OfflineProcessing +java -cp ./console/target/device-detection-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar --enable-native-access=ALL-UNNAMED fiftyone.devicedetection.examples.console.OfflineProcessing ``` With the Maven exec plugin, use the `exec:exec` goal. `exec:java` runs in the Maven JVM @@ -162,3 +165,15 @@ unnamed module. To grant native access to 51Degrees code alone you need the indi jars, with `pipeline.engines.fiftyone` and `device-detection.hash.engine.on-premise` on the module path - see the [device-detection-java README](https://github.com/51Degrees/51degrees-java/blob/main/device-detection/README.md#native-library-access). + +## Flow Data and threads + +A Flow Data cannot be used by two threads at once. The web examples handle +each request on one thread, and read the Flow Data there. An application that +hands work from a request to another thread or task reads the values that +work needs from the Flow Data on the request's thread, into its own +variables, and passes those values, not the Flow Data. The Flow Data is +closed when the request ends, so work that outlives the request must not +keep it either. The +[pipeline README](../../pipeline/README.md#flow-data-and-threads) +shows such a hand-off. diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java index 846e1fcb4..fa8884129 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedCloud.java @@ -22,6 +22,7 @@ package fiftyone.devicedetection.examples.console; +import fiftyone.devicedetection.examples.shared.FindOutMore; import fiftyone.devicedetection.DeviceDetectionPipelineBuilder; import fiftyone.devicedetection.examples.shared.KeyHelper; import fiftyone.devicedetection.examples.shared.EvidenceHelper; @@ -47,25 +48,45 @@ *

* In order to use the cloud service you will need to obtain a "Resource Key". A free resource key * configured with the properties required by this example may be obtained from - * https://configure.51degrees.com/jqz435Nc?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedcloud.java&utm_term=gettingstartedcloud-2 + * https://51degrees.com/configure/jqz435Nc?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedcloud.java&utm_term=gettingstartedcloud-2 *

* The concepts of "pipeline", "flow data", "evidence" and "results" are illustrated. */ @SuppressWarnings("DuplicatedCode") public class GettingStartedCloud { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedcloud.java"; + private static final Logger logger = LoggerFactory.getLogger(GettingStartedCloud.class); /** - * The resource key can be supplied as an argument to this program or as an environment - * variable or as a Java system property called "TestResourceKey". + * The resource key can be supplied as an argument to this program, or + * as an environment variable or Java system property called + * "_51DEGREES_RESOURCE_KEY". With none, + * the example says how to get one and ends. */ public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); - String resourceKey = args.length > 0 ? args[0]: null; + String resourceKey = KeyHelper.findResourceKey( + args.length > 0 ? args[0] : null, + KeyHelper.RESOURCE_KEY_VARIABLE); + if (resourceKey == null) { + KeyHelper.writeNoResourceKey(System.out, EXAMPLE_TAG, + KeyHelper.RESOURCE_KEY_VARIABLE, + KeyHelper.FREE_PROPERTIES_CONFIGURATION); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); + return; + } // prepare 'evidence' for use in pipeline (see below) List> evidence = EvidenceHelper.setUpEvidence(); run(resourceKey, evidence, System.out); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); } /** diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java index 6cb6e8433..0620c2562 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java @@ -22,6 +22,7 @@ package fiftyone.devicedetection.examples.console; +import fiftyone.devicedetection.examples.shared.FindOutMore; import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.devicedetection.examples.shared.EvidenceHelper; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; @@ -58,6 +59,12 @@ * The concepts of "pipeline", "flow data", "evidence" and "results" are illustrated. */ public class GettingStartedOnPrem { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedonprem.java"; + private static final Logger logger = LoggerFactory.getLogger(GettingStartedOnPrem.class); /* In this example, by default, the 51degrees "Lite" file is used from the data repository in @@ -84,6 +91,8 @@ public static void main(String[] args) throws Exception { // prepare 'evidence' for use in pipeline (see below) List> evidence = EvidenceHelper.setUpEvidence(); run(dataFile, evidence, System.out); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); } /** @@ -119,7 +128,11 @@ public static void run(String dataFile, // load the options and if no resource key has been set in the file // use the one supplied to this method PipelineOptions pipelineOptions = PipelineOptionsFactory.getOptionsFromFile(optionsFile); - // Build a new Pipeline from the configuration. + /* Build a new Pipeline from the configuration. The options file sets + ShareUsage to false, because this example processes evidence 51Degrees + already holds, but in normal operation usage sharing should be enabled + by adding a Usage Sharing Element to the Pipeline, which the + FiftyOnePipelineBuilder does unless ShareUsage is false. */ try (Pipeline pipeline = new FiftyOnePipelineBuilder() .buildFromConfiguration(pipelineOptions)) { diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ManualDataUpdate.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ManualDataUpdate.java new file mode 100644 index 000000000..68a9447e7 --- /dev/null +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ManualDataUpdate.java @@ -0,0 +1,204 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.console; + +import fiftyone.devicedetection.DeviceDetectionPipelineBuilder; +import fiftyone.devicedetection.examples.shared.DataFileHelper; +import fiftyone.devicedetection.examples.shared.FindOutMore; +import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; +import fiftyone.devicedetection.shared.DeviceData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.Constants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.OutputStream; +import java.io.PrintWriter; +import java.text.SimpleDateFormat; + +import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; +import static fiftyone.devicedetection.examples.shared.PropertyHelper.asString; +import static fiftyone.pipeline.util.FileFinder.getFilePath; + +/** + * Shows how to make the on-premise device detection engine load a new data + * file when the application decides to, rather than through an automatic + * update. The engine goes on detecting while the file loads, so the + * application does not need to restart. + *

+ * In practice a newer data file is copied over the one the engine was built + * with, for example by a scheduled task, and then refreshData is called. + * This example has no newer file, so it reloads the file it started with + * and the results stay the same. refreshData can also be given the + * contents of a data file held in memory. + */ +public class ManualDataUpdate { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-devicedetection-examples-console-manualdataupdate.java"; + + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-manualdataupdate.java&utm_term=enterprise-data-file"; + + private static final String DESKTOP_USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36"; + + private static final String IPHONE_USER_AGENT = + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) " + + "AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114"; + + private static final Logger logger = + LoggerFactory.getLogger(ManualDataUpdate.class); + + public static void main(String[] args) throws Exception { + configureLogback(getFilePath("logback.xml")); + // The free Lite data file is used unless another is given. + String dataFile = args.length > 0 ? + args[0] : GettingStartedOnPrem.LITE_V_4_1_HASH; + run(dataFile, System.out); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); + } + + /** + * Run the example. + * @param dataFile a device detection data file + * @param outputStream somewhere for the results + */ + public static void run(String dataFile, OutputStream outputStream) + throws Exception { + String dataFileLocation; + try { + dataFileLocation = + DataFileHelper.findDataFile(dataFile).getAbsolutePath(); + } catch (Exception e) { + DataFileHelper.cantFindDataFile(dataFile); + throw e; + } + PrintWriter writer = new PrintWriter(outputStream, true); + logger.info("Constructing pipeline from file " + dataFileLocation); + + // The data file changes only when this code says so, because + // automatic updates and the file system watcher are off. + try (Pipeline pipeline = new DeviceDetectionPipelineBuilder() + // The engine works from a copy of the data file, so a newer + // file can be copied over the original while it runs. + .useOnPremise(dataFileLocation, true) + .setAutoUpdate(false) + .setDataFileSystemWatcher(false) + .setPerformanceProfile(Constants.PerformanceProfiles.LowMemory) + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. + .setShareUsage(false) + .build()) { + DeviceDetectionHashEngine engine = + pipeline.getElement(DeviceDetectionHashEngine.class); + + boolean needsEnterprise = false; + needsEnterprise |= checkIfMobile(pipeline, engine, + DESKTOP_USER_AGENT, writer); + needsEnterprise |= checkIfMobile(pipeline, engine, + IPHONE_USER_AGENT, writer); + + writer.println("Reloading the data file published on " + + published(engine) + "."); + // Load the data file again, from the place the engine was built + // with. Call this once a newer data file has been copied there. + engine.refreshData(engine.getDataFileMetaData().getIdentifier()); + writer.println("Reloaded the data file published on " + + published(engine) + "."); + writer.println(); + + needsEnterprise |= checkIfMobile(pipeline, engine, + DESKTOP_USER_AGENT, writer); + needsEnterprise |= checkIfMobile(pipeline, engine, + IPHONE_USER_AGENT, writer); + + if (needsEnterprise) { + writer.println("This example reloaded the free Lite data " + + "file. The Enterprise data file adds daily automatic " + + "updates."); + DataFileHelper.writeEnterpriseNote(writer, CONTACT_US); + } + } + writer.flush(); + } + + /** + * Detect the device a User-Agent comes from and write whether it is a + * mobile device, and its hardware vendor, which only the Enterprise data + * file has. + * @return true if the hardware vendor needs the Enterprise data file + */ + private static boolean checkIfMobile(Pipeline pipeline, + DeviceDetectionHashEngine engine, + String userAgent, + PrintWriter writer) + throws Exception { + // The flow data is closed straight away, which releases the results + // of the detection. Results that are not released keep the data + // they were read from in memory, even after the data is refreshed. + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", userAgent); + data.process(); + DeviceData device = data.get(DeviceData.class); + writer.println("Is User-Agent '" + userAgent + + "' a mobile device? " + asString(device.getIsMobile())); + if (DataFileHelper.hasProperty(engine, "HardwareVendor")) { + writer.println("\tHardware Vendor: " + + asString(device.getHardwareVendor())); + writer.println(); + return false; + } + writer.println("\tHardware Vendor: " + + DataFileHelper.NEEDS_ENTERPRISE); + writer.println(); + return true; + } + } + + /** + * The date the engine's data file was published. + */ + private static String published(DeviceDetectionHashEngine engine) { + return new SimpleDateFormat("yyyy-MM-dd").format( + engine.getDataFileMetaData().getDataPublishedDateTime()); + } +} +/*! + * @example console/ManualDataUpdate.java + * This example shows how to make the on-premise device detection engine + * load a new data file when the application decides to, rather than through + * an automatic update. + *

+ * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ManualDataUpdate.java). + * @include{doc} example-require-datafile.txt + */ diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java index ddabcaed4..ae753978d 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java @@ -22,6 +22,7 @@ package fiftyone.devicedetection.examples.console; +import fiftyone.devicedetection.examples.shared.FindOutMore; import fiftyone.devicedetection.DeviceDetectionPipelineBuilder; import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.devicedetection.examples.shared.EvidenceHelper; @@ -45,7 +46,7 @@ import java.util.Objects; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; +import static fiftyone.devicedetection.examples.shared.DataFileHelper.LITE_HASH_DATA_FILE_PATH; import static fiftyone.pipeline.util.FileFinder.getFilePath; import static java.util.stream.Collectors.groupingBy; @@ -61,6 +62,12 @@ * page. */ public class MatchMetrics { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-devicedetection-examples-console-matchmetrics.java"; + private static final Logger logger = LoggerFactory.getLogger(MatchMetrics.class); /** @@ -76,6 +83,8 @@ public static void main(String[] args) throws Exception { String dataFilename = DataFileHelper.getDataFileLocation( args.length > 0 ? args[0] : LITE_HASH_DATA_FILE_PATH); run(dataFilename, EvidenceHelper.setUpEvidence(), false, System.out); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); } /** diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataCloud.java index 8ca360fab..875a7fb0a 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataCloud.java @@ -21,6 +21,7 @@ * ********************************************************************* */ package fiftyone.devicedetection.examples.console; +import fiftyone.devicedetection.examples.shared.FindOutMore; import fiftyone.devicedetection.DeviceDetectionPipelineBuilder; import fiftyone.devicedetection.cloud.flowelements.DeviceDetectionCloudEngine; import fiftyone.devicedetection.examples.shared.KeyHelper; @@ -42,12 +43,36 @@ public class MetadataCloud { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-devicedetection-examples-console-metadatacloud.java"; + private static final Logger logger = LoggerFactory.getLogger(GettingStartedOnPrem.class); + /** + * The resource key can be supplied as an argument to this program, or + * as an environment variable or Java system property called + * "_51DEGREES_RESOURCE_KEY". With none, + * the example says how to get one and ends. + */ public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); - String resourceKey = args.length > 0 ? args[0]: null; + String resourceKey = KeyHelper.findResourceKey( + args.length > 0 ? args[0] : null, + KeyHelper.RESOURCE_KEY_VARIABLE); + if (resourceKey == null) { + KeyHelper.writeNoResourceKey(System.out, EXAMPLE_TAG, + KeyHelper.RESOURCE_KEY_VARIABLE, + KeyHelper.FREE_PROPERTIES_CONFIGURATION); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); + return; + } run(resourceKey, System.out); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); } diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java index c85d852b8..a7dbdb5f1 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java @@ -21,6 +21,7 @@ * ********************************************************************* */ package fiftyone.devicedetection.examples.console; +import fiftyone.devicedetection.examples.shared.FindOutMore; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; @@ -45,6 +46,12 @@ public class MetadataOnPrem { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-devicedetection-examples-console-metadataonprem.java"; + private static final Logger logger = LoggerFactory.getLogger(GettingStartedOnPrem.class); /* In this example, by default, the 51degrees "Lite" file is used from the data repository in @@ -65,6 +72,8 @@ public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); String dataFile = args.length > 0 ? args[0] : LITE_V_4_1_HASH; run(dataFile, System.out); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); } diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MinimalExample.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MinimalExample.java index 8c5fb5206..6ea23a441 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MinimalExample.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MinimalExample.java @@ -22,13 +22,15 @@ package fiftyone.devicedetection.examples.console; +import fiftyone.devicedetection.examples.shared.FindOutMore; +import fiftyone.devicedetection.examples.shared.KeyHelper; import fiftyone.devicedetection.DeviceDetectionPipelineBuilder; import fiftyone.devicedetection.shared.DeviceData; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.Pipeline; /** - * This is the code for the minimal example displayed in the configurator. Please see + * This is the code for the minimal example displayed on the configuration page. Please see * * Getting Started Console Cloud for a fuller example. */ @@ -44,9 +46,27 @@ */ public class MinimalExample { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-devicedetection-examples-console-minimalexample.java"; + public static void main(String[] args) throws Exception { - String resource = args.length > 0 ? args[0] : null; - run(resource); + String resourceKey = KeyHelper.findResourceKey( + args.length > 0 ? args[0] : null, + KeyHelper.RESOURCE_KEY_VARIABLE); + if (resourceKey == null) { + KeyHelper.writeNoResourceKey(System.out, EXAMPLE_TAG, + KeyHelper.RESOURCE_KEY_VARIABLE, + KeyHelper.FREE_PROPERTIES_CONFIGURATION); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); + return; + } + run(resourceKey); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); } public static void run(String resource) throws Exception { diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java index 3c6c724ac..12900fa9d 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java @@ -22,6 +22,8 @@ package fiftyone.devicedetection.examples.console; +import fiftyone.devicedetection.examples.shared.FindOutMore; +import fiftyone.devicedetection.examples.shared.KeyHelper; import fiftyone.devicedetection.cloud.data.MultiDeviceDataCloud; import fiftyone.devicedetection.cloud.flowelements.HardwareProfileCloudEngineBuilder; import fiftyone.devicedetection.shared.Constants; @@ -41,14 +43,30 @@ import static fiftyone.pipeline.util.FileFinder.getFilePath; public class NativeModelCloud { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-devicedetection-examples-console-nativemodelcloud.java"; + // Example values to use when looking up device details from native model names. private static final String NATIVE_MODEL_1 = "SC-03L"; private static final String NATIVE_MODEL_2 = "iPhone11,8"; - public static final String NATIVE_MODEL_EXAMPLE_RESOURCE_KEY_NAME = "SuperResourceKey"; + public static final String NATIVE_MODEL_EXAMPLE_RESOURCE_KEY_NAME = KeyHelper.SUPER_RESOURCE_KEY_VARIABLE; public static void main (String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); - String resourceKey = args.length > 0 ? args[0] : null; + String resourceKey = KeyHelper.findResourceKey( + args.length > 0 ? args[0] : null, + NATIVE_MODEL_EXAMPLE_RESOURCE_KEY_NAME); + if (resourceKey == null) { + KeyHelper.writeNoResourceKey(System.out, EXAMPLE_TAG, + NATIVE_MODEL_EXAMPLE_RESOURCE_KEY_NAME, + KeyHelper.PAID_PROPERTIES_CONFIGURATION); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); + return; + } run(resourceKey, System.out); } @@ -90,6 +108,10 @@ public static void run(String resourceKey, OutputStream os) throws Exception { // Repeat for an alternative native model name. analyseNativeModel(NATIVE_MODEL_2, pipeline, output); } + // The writer closes the stream it was given, so the section is + // written before it. + FindOutMore.write(output, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); } } @@ -131,7 +153,7 @@ static void analyseNativeModel(String nativemodel, Pipeline pipeline, PrintWrite * * Unlike other examples, use of this example requires a license key which can be purchased from [our * pricing page](https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-nativemodelcloud.java&utm_term=top). Once this is done, a resource key with the - * properties required by this example can be created at [here](https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-nativemodelcloud.java&utm_term=top). + * properties required by this example can be created at [here](https://51degrees.com/configure/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-nativemodelcloud.java&utm_term=top). * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/NativeModelCloud.java). */ \ No newline at end of file diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java index fbcdee51b..173d74ba0 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java @@ -22,6 +22,7 @@ package fiftyone.devicedetection.examples.console; +import fiftyone.devicedetection.examples.shared.FindOutMore; import fiftyone.devicedetection.DeviceDetectionPipelineBuilder; import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; @@ -67,6 +68,12 @@ * predictive power using Performance Profile, Graph and Difference and Drift settings. */ public class OfflineProcessing { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-devicedetection-examples-console-offlineprocessing.java"; + static final Logger logger = LoggerFactory.getLogger(OfflineProcessing.class); // This 51degrees "Lite" file is used from the data repository in // cxx/data, and is downloaded if it is missing @@ -238,6 +245,10 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws } // finish the last YAML document writer.write("...\n"); + // The section is written as comments, so the output stays a + // valid YAML stream. + FindOutMore.writeCommented(new PrintWriter(writer), "# ", + EXAMPLE_TAG, FindOutMore.ON_PREMISE_REPOSITORIES); writer.flush(); logger.info("Finished processing {} records", count); } diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java index 163e9f8da..3a527d479 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java @@ -22,6 +22,7 @@ package fiftyone.devicedetection.examples.console; +import fiftyone.devicedetection.examples.shared.FindOutMore; import fiftyone.common.testhelpers.LogbackHelper; import fiftyone.devicedetection.DeviceDetectionOnPremisePipelineBuilder; import fiftyone.devicedetection.DeviceDetectionPipelineBuilder; @@ -46,7 +47,7 @@ import static fiftyone.devicedetection.examples.shared.DataFileHelper.getDataFileLocation; import static fiftyone.devicedetection.examples.shared.DataFileHelper.getEvidenceFile; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; +import static fiftyone.devicedetection.examples.shared.DataFileHelper.LITE_HASH_DATA_FILE_PATH; import static fiftyone.pipeline.engines.Constants.PerformanceProfiles.*; /** @@ -58,6 +59,12 @@ * for more information about adjusting performance. */ public class PerformanceBenchmark { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-devicedetection-examples-console-performancebenchmark.java"; + // the default number of threads if one is not provided. public static final int DEFAULT_NUMBER_OF_THREADS = 4; // the number of tests to execute. @@ -106,6 +113,8 @@ public static void main(String[] args) throws Exception { evidenceFilename, numberOfThreads, new PrintWriter(System.out,true)); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); } /** diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java index 50d44c364..e95d71faf 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/ShareUsageExample.java @@ -22,6 +22,7 @@ package fiftyone.devicedetection.examples.console; +import fiftyone.devicedetection.examples.shared.FindOutMore; import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.Pipeline; @@ -37,6 +38,7 @@ import java.nio.file.Files; import java.util.HashMap; import java.util.Iterator; +import java.util.Locale; import java.util.Map; import java.util.Random; import java.util.stream.Collectors; @@ -74,6 +76,12 @@ * identifier before running this example.

*/ public class ShareUsageExample { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-devicedetection-examples-console-shareusageexample.java"; + static final Logger logger = LoggerFactory.getLogger(ShareUsageExample.class); // Evidence file with 20,000 examples, from the data repository in @@ -104,6 +112,8 @@ public static void main(String[] args) throws Exception { File evidenceFile = DataFileHelper.findDataFile(HEADER_EVIDENCE_YML); run(Files.newInputStream(evidenceFile.toPath())); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); } /** @@ -111,8 +121,21 @@ public static void main(String[] args) throws Exception { * * @param is an InputStream containing YAML documents with evidence */ - @SuppressWarnings("unchecked") public static void run(InputStream is) throws Exception { + run(is, null); + } + + /** + * Process evidence from a YAML input stream and share it via the + * ShareUsageElement, sending it to the endpoint given. + * + * @param is an InputStream containing YAML documents with evidence + * @param shareUsageUrl where to send the usage data, or null for the + * 51Degrees endpoint. A test gives an endpoint of + * its own, so that it shares nothing. + */ + @SuppressWarnings("unchecked") + public static void run(InputStream is, String shareUsageUrl) throws Exception { logger.info("Starting ShareUsage example"); logger.info("Settings: minEntriesPerMessage={}, sharePercentage={}%, recordsToProcess={}", MINIMUM_ENTRIES_PER_MESSAGE, @@ -120,7 +143,12 @@ public static void run(InputStream is) throws Exception { RECORDS_TO_PROCESS); // Build the ShareUsageElement with custom settings - ShareUsageElement shareUsageElement = new ShareUsageBuilder(LoggerFactory.getILoggerFactory()) + ShareUsageBuilder builder = new ShareUsageBuilder(LoggerFactory.getILoggerFactory()); + if (shareUsageUrl != null) { + // Send to another endpoint than the 51Degrees one + builder.setShareUsageUrl(shareUsageUrl); + } + ShareUsageElement shareUsageElement = builder // Set the minimum number of entries before sending a batch .setMinimumEntriesPerMessage(MINIMUM_ENTRIES_PER_MESSAGE) // Set the maximum queue size (must be > minEntriesPerMessage) @@ -215,8 +243,8 @@ private static Map filterEvidence(Map evidence, * * @return a random IPv4 address string */ - private static String generateRandomIp() { - return String.format("%d.%d.%d.%d", + static String generateRandomIp() { + return String.format(Locale.ROOT, "%d.%d.%d.%d", random.nextInt(256), random.nextInt(256), random.nextInt(256), diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java index ee5ee1e0b..40029b2f2 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java @@ -22,10 +22,11 @@ package fiftyone.devicedetection.examples.console; +import fiftyone.devicedetection.examples.shared.FindOutMore; +import fiftyone.devicedetection.examples.shared.KeyHelper; import fiftyone.devicedetection.cloud.data.MultiDeviceDataCloud; import fiftyone.devicedetection.shared.Constants; import fiftyone.devicedetection.shared.DeviceData; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; import fiftyone.pipeline.core.configuration.PipelineOptions; import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; import fiftyone.pipeline.core.data.FlowData; @@ -49,21 +50,41 @@ * Unlike other examples, use of this example requires a license key which can be purchased from our * pricing page. Once this is done, a resource key with the * properties required by this example can be created - * here. + * here. *

* This resource key can be supplied as a command line argument to this program, as an environment - * variable "SuperResourceKey", as a System Property "SuperResourceKey" or by editing the config + * variable "_51DEGREES_RESOURCE_KEY_SUPER", as a System Property "_51DEGREES_RESOURCE_KEY_SUPER" or by editing the config * file tacCloud.xml. */ public class TacCloud { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-devicedetection-examples-console-taccloud.java"; + // Example values to use when looking up device details from TACs. static final String TAC_1 = "35925406"; static final String TAC_2 = "86386802"; - public static final String TAC_EXAMPLE_RESOURCE_KEY_NAME = "SuperResourceKey"; + public static final String TAC_EXAMPLE_RESOURCE_KEY_NAME = KeyHelper.SUPER_RESOURCE_KEY_VARIABLE; public static void main (String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); - String resourceKey = args.length > 0 ? args[0] : null; + String resourceKey = KeyHelper.findResourceKey( + args.length > 0 ? args[0] : null, + TAC_EXAMPLE_RESOURCE_KEY_NAME); + // A key typed into the options file is used as it is. + String fileResourceKey = PipelineOptionsFactory.getOptionsFromFile( + getFilePath("tacCloud.xml")) + .find("CloudRequestEngine", "ResourceKey"); + if (resourceKey == null && isInvalidKey(fileResourceKey)) { + KeyHelper.writeNoResourceKey(System.out, EXAMPLE_TAG, + TAC_EXAMPLE_RESOURCE_KEY_NAME, + KeyHelper.PAID_PROPERTIES_CONFIGURATION); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); + return; + } run(resourceKey, System.out); } @@ -80,7 +101,7 @@ public static void run(String resourceKey, OutputStream os) throws Exception { // use the one supplied to this method PipelineOptions pipelineOptions = PipelineOptionsFactory.getOptionsFromFile(optionsFile); String fileResourceKey = pipelineOptions.find("CloudRequestEngine", "ResourceKey"); - if (KeyUtils.isInvalidKey(fileResourceKey)) { + if (isInvalidKey(fileResourceKey)) { getOrSetSuperResourceKey(resourceKey, TAC_EXAMPLE_RESOURCE_KEY_NAME); } try (PrintWriter writer = new PrintWriter(os)) { @@ -100,6 +121,10 @@ public static void run(String resourceKey, OutputStream os) throws Exception { analyseTac(TAC_1, pipeline, writer); analyseTac(TAC_2, pipeline, writer); } + // The writer closes the stream it was given, so the section is + // written before it. + FindOutMore.write(writer, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); } } @@ -136,7 +161,7 @@ static void analyseTac(String tac, Pipeline pipeline, PrintWriter output) throws * * Unlike other examples, use of this example requires a license key which can be purchased from our * [pricing page](https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-taccloud.java&utm_term=top). Once this is done, a resource key with the - * properties required by this example can be created [here](https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-taccloud.java&utm_term=top). + * properties required by this example can be created [here](https://51degrees.com/configure/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-taccloud.java&utm_term=top). * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/TacCloud.java). * diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java index ebffdef86..93921afa4 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java @@ -22,10 +22,15 @@ package fiftyone.devicedetection.examples.console; +import fiftyone.devicedetection.examples.shared.FindOutMore; import fiftyone.devicedetection.DeviceDetectionPipelineBuilder; import fiftyone.devicedetection.examples.shared.DataFileHelper; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; +import fiftyone.devicedetection.examples.shared.KeyHelper; +import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; import fiftyone.pipeline.engines.services.DataUpdateService; import fiftyone.pipeline.engines.services.DataUpdateServiceDefault; import fiftyone.pipeline.engines.services.OnUpdateComplete; @@ -37,13 +42,14 @@ import java.io.PrintWriter; import java.nio.file.*; import java.text.SimpleDateFormat; +import java.util.Locale; import java.util.Objects; import java.util.Scanner; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; import static fiftyone.devicedetection.examples.shared.DataFileHelper.getDatafileMetaData; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.ENTERPRISE_HASH_DATA_FILE_NAME; -import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; +import static fiftyone.devicedetection.examples.shared.DataFileHelper.ENTERPRISE_HASH_DATA_FILE_NAME; +import static fiftyone.devicedetection.examples.shared.DataFileHelper.LITE_HASH_DATA_FILE_PATH; import static fiftyone.pipeline.util.FileFinder.getFilePath; /*! @@ -52,11 +58,12 @@ * device detection engine, and controls when a new data file is sought and when it is loaded by * the device detection software. *

- * Three main aspects are demonstrated: + * Four main aspects are demonstrated: *

    *
  1. Update on Start-Up
  2. *
  3. Filesystem Watcher
  4. *
  5. Daily auto-update
  6. + *
  7. Programmatic update
  8. *
* ## License Key * Updating the data file needs a 51Degrees licence key for the Enterprise data file. Without @@ -164,6 +171,16 @@ * ``` {java} .setUpdateRandomisationMax(10*60) * ``` + * ## Programmatic update + * Your own code can ask for an update at a time it chooses, for example from a scheduled job + * with automatic updates turned off. The data update service checks for a newer data file, + * downloads it and refreshes the engine. + * ``` {java} + * DataUpdateService.AutoUpdateStatus status = dataUpdateService.checkForUpdate( + * pipeline.getElement(DeviceDetectionHashEngine.class)); + * ``` + * To load a data file that your own process has put in place, call the engine's + * `refreshData(null)` instead. * # Location * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java). * @@ -175,11 +192,12 @@ * device detection engine, and controls when a new data file is sought and when it is loaded by * the device detection software. *

- * Three main aspects are demonstrated: + * Four main aspects are demonstrated: *

    *
  1. Update on Start-Up
  2. *
  3. Filesystem Watcher
  4. *
  5. Daily auto-update
  6. + *
  7. Programmatic update
  8. *
*

* To update the data file you need a licence key, which you can ask for on our @@ -189,7 +207,13 @@ */ public class UpdateDataFile { - public static final String UPDATE_EXAMPLE_LICENSE_KEY_NAME = "LicenseKey"; + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-devicedetection-examples-console-updatedatafile.java"; + + public static final String UPDATE_EXAMPLE_LICENSE_KEY_NAME = "_51DEGREES_LICENSE_KEY"; public static final String DEFAULT_DATA_FILENAME = System.getProperty("user.dir") + File.separator + ENTERPRISE_HASH_DATA_FILE_NAME; @@ -209,6 +233,8 @@ public static void main(String[] args) throws Exception { String licenseKey = args.length > 0 ? args[0] : null; String dataFilename = args.length > 1 ? args[1] : null; run(dataFilename, licenseKey, true); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); } /** @@ -227,9 +253,9 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr // try to find a license key if (Objects.isNull(licenseKey)) { - licenseKey = KeyUtils.getNamedKey(UPDATE_EXAMPLE_LICENSE_KEY_NAME); + licenseKey = KeyHelper.getNamedKey(UPDATE_EXAMPLE_LICENSE_KEY_NAME); } - if (Objects.isNull(licenseKey) || KeyUtils.isInvalidKey(licenseKey)) { + if (Objects.isNull(licenseKey) || KeyHelper.isInvalidKey(licenseKey)) { explainLicenseKey(dataFilename, new PrintWriter(System.out, true)); return; } @@ -277,7 +303,7 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr "in your licensed allocation."); System.out.print("Do you wish to continue with this example (y)? "); String input = scanner.nextLine(); - if ((input.isEmpty() || input.toLowerCase().startsWith("y")) == false) { + if ((input.isEmpty() || input.toLowerCase(Locale.ROOT).startsWith("y")) == false) { logger.info("Stopping example without download"); return; } @@ -359,11 +385,40 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr completionListener.args.getStatus()); } + // Programmatic (non-automatic) update: ask for an update now, + // rather than wait for the next scheduled check. The file + // just loaded is the newest, so the status is normally + // AUTO_UPDATE_NOT_NEEDED. + DeviceDetectionHashEngine engine = + pipeline.getElement(DeviceDetectionHashEngine.class); + logger.info("Programmatic update complete, status: {}", + updateNow(dataUpdateService, engine)); + logger.info("Finished Example"); } } } + /** + * A programmatic (non-automatic) update. The data update service checks + * now for a newer data file for the engine, downloads it if there is + * one, and refreshes the engine with it, as an automatic update does. + * Call this when your own code decides it is time to update, for + * example from a scheduled job, with automatic updates turned off. + * To load a file that your own process has put in place, call the + * engine's {@code refreshData(null)} instead. + * @param dataUpdateService the service the pipeline was built with + * @param engine the on-premise engine to update + * @return the status of the update, such as AUTO_UPDATE_SUCCESS, or + * AUTO_UPDATE_NOT_NEEDED when the file in use is the newest + */ + static DataUpdateService.AutoUpdateStatus updateNow( + DataUpdateService dataUpdateService, + OnPremiseAspectEngine engine) { + return dataUpdateService.checkForUpdate(engine); + } + /** * Without a licence key the example cannot download a data file, so it * shows the data file it would start from, the free Lite file unless diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/comparison/Comparer.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/comparison/Comparer.java index ba231800a..5f4c7989e 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/comparison/Comparer.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/comparison/Comparer.java @@ -22,6 +22,7 @@ package fiftyone.devicedetection.examples.console.comparison; +import fiftyone.devicedetection.examples.shared.FindOutMore; import fiftyone.common.testhelpers.LogbackHelper; import fiftyone.devicedetection.examples.console.comparison.Detection.BenchmarkResult; import fiftyone.devicedetection.examples.console.comparison.Detection.Request; @@ -39,8 +40,12 @@ import static fiftyone.devicedetection.examples.shared.DataFileHelper.getEvidenceFile; /** - * Unlike other examples, use of this example requires a 51Degrees Enterprise data file, which can - * be obtained on a trial basis or purchased from our pricing page here. + * The 51Degrees solution uses the free Lite data file, downloaded if it is + * missing, unless the first argument names another data file. The Lite data + * file has only some of the properties the comparison reads, so after the + * report the example says what the Enterprise data file adds and where to ask + * for a copy. Give the Enterprise data file as the first argument to compare + * with every property. *

* This class provides a framework and runner for comparing different device detection solutions. * It is provided in a basic form to compare detection time. Different solutions may be compared @@ -50,6 +55,12 @@ * code has not been tested and it is left to those interested to uncomment and get working. */ public class Comparer { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-devicedetection-examples-console-comparison-comparer.java"; + public static final int DEFAULT_NUMBER_OF_THREADS = 4; public static final int DEFAULT_NUMBER_OF_RESULTS = 2500; @SuppressWarnings("resource") @@ -64,11 +75,19 @@ public class Comparer { public static void main(String[] args) throws Exception { LogbackHelper.configureLogback(FileFinder.getFilePath("logback.xml")); - new Comparer().compare(Arrays.asList(DEFAULT_SOLUTIONS), + @SuppressWarnings("resource") + List solutions = args.length > 0 ? + Arrays.asList( + new DetectionImplFiftyOneDegrees.FiftyOneSolution(args[0]), + new DetectionImplBrowsCap.BrowsCapSolution()) : + Arrays.asList(DEFAULT_SOLUTIONS); + new Comparer().compare(solutions, DEFAULT_NUMBER_OF_THREADS, DEFAULT_NUMBER_OF_RESULTS, new Reporting.Minimal(), new PrintWriter(System.out, true)); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); } /** @@ -115,6 +134,9 @@ public void compare(List solutions, } logger.info("Preparing reports"); reportingModel.report(executionResults, writer); + for (Solution solution : solutions) { + solution.writeNote(writer); + } logger.info("Comparison done"); } diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/comparison/Detection.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/comparison/Detection.java index 645eff02b..47dc2afd9 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/comparison/Detection.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/comparison/Detection.java @@ -22,6 +22,7 @@ package fiftyone.devicedetection.examples.console.comparison; +import java.io.PrintWriter; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -147,5 +148,13 @@ interface Solution extends AutoCloseable { Properties detect(Detection.Request request) throws Exception; String getVendorId(); + + /** + * Writes anything the reader should know about how the solution + * ran, after the report. Nothing by default. + * @param writer where to write + */ + default void writeNote(PrintWriter writer) { + } } } diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/comparison/DetectionImplFiftyOneDegrees.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/comparison/DetectionImplFiftyOneDegrees.java index 4e50f8bf0..3057e03ee 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/comparison/DetectionImplFiftyOneDegrees.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/comparison/DetectionImplFiftyOneDegrees.java @@ -22,43 +22,98 @@ package fiftyone.devicedetection.examples.console.comparison; +import fiftyone.devicedetection.DeviceDetectionOnPremisePipelineBuilder; import fiftyone.devicedetection.DeviceDetectionPipelineBuilder; +import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.devicedetection.examples.shared.PropertyHelper; +import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.shared.DeviceData; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.Constants; -import fiftyone.pipeline.util.FileFinder; +import fiftyone.pipeline.engines.data.AspectPropertyValue; -import java.io.FileNotFoundException; +import java.io.PrintWriter; +import java.util.Collections; +import java.util.HashSet; import java.util.Objects; +import java.util.Set; +import java.util.function.Supplier; /** - * Implementation of {@link Detection} for 51Degrees. + * Implementation of {@link Detection} for 51Degrees. It uses the free Lite + * data file unless it is given another, such as the Enterprise data file, + * which has every property it reads. */ public class DetectionImplFiftyOneDegrees { public static final String FIFTY_ONE_DEGREES = "FiftyOneDegrees"; - public static final String ENTERPRISE_HASH_DATA_FILE_NAME = "Enterprise-HashV41.hash"; + + /** + * The free data file, downloaded if it is missing. + */ + public static final String LITE_HASH_DATA_FILE_NAME = + "device-detection/51Degrees-LiteV4.1.hash"; + + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * file. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-comparison-detectionimplfiftyonedegrees.java&utm_term=enterprise-data-file"; + + /** + * The properties this solution reads. + */ + private static final String[] PROPERTIES = { + "HardwareVendor", "HardwareModel", "BrowserVendor", + "BrowserVersion", "IsMobile", "DeviceType"}; + public static class FiftyOneConfig { String dataFile; - public FiftyOneConfig() throws IllegalArgumentException { - dataFile = FileFinder.getFilePath(ENTERPRISE_HASH_DATA_FILE_NAME).getAbsolutePath(); + public FiftyOneConfig() { + this(LITE_HASH_DATA_FILE_NAME); + } + + /** + * @param dataFile the data file to use, found as the other examples + * find theirs + */ + public FiftyOneConfig(String dataFile) { + this.dataFile = + DataFileHelper.findDataFile(dataFile).getAbsolutePath(); } } public static class FiftyOneProperties extends Detection.Properties.Base { - FiftyOneProperties(Detection.Request request, FlowData flowData) { + /** + * @param missing the properties the data file does not have, which + * read as needing the Enterprise data file + */ + FiftyOneProperties(Detection.Request request, FlowData flowData, + Set missing) { super(request); // transcribe properties so flowData can be freed DeviceData device = flowData.get(DeviceData.class); - isMobile = PropertyHelper.asString(device.getIsMobile()); - hardwareVendor = PropertyHelper.asString(device.getHardwareVendor()); - hardwareModel = PropertyHelper.asString(device.getHardwareModel()); - browserVendor = PropertyHelper.asString(device.getBrowserVendor()); - browserVersion = PropertyHelper.asString(device.getBrowserVersion()); - deviceType = PropertyHelper.asString(device.getDeviceType()); + isMobile = valueOf(missing, "IsMobile", device::getIsMobile); + hardwareVendor = valueOf(missing, "HardwareVendor", + device::getHardwareVendor); + hardwareModel = valueOf(missing, "HardwareModel", + device::getHardwareModel); + browserVendor = valueOf(missing, "BrowserVendor", + device::getBrowserVendor); + browserVersion = valueOf(missing, "BrowserVersion", + device::getBrowserVersion); + deviceType = valueOf(missing, "DeviceType", device::getDeviceType); + } + + private static String valueOf( + Set missing, + String name, + Supplier> value) { + return missing.contains(name) ? + DataFileHelper.NEEDS_ENTERPRISE : + PropertyHelper.asString(value.get()); } @Override @@ -70,38 +125,73 @@ public String getVendorId() { public static class FiftyOneSolution implements Detection.Solution { Pipeline pipeline; FiftyOneConfig config; + private final String dataFile; + private Set missing = Collections.emptySet(); + private boolean lite; + public FiftyOneSolution() { + this(LITE_HASH_DATA_FILE_NAME); + } + + /** + * @param dataFile the data file to use + */ + public FiftyOneSolution(String dataFile) { + this.dataFile = dataFile; + } @Override public void initialise(int numberOfThreads) throws Exception { - try { - this.config = new FiftyOneConfig(); - } catch (IllegalArgumentException e) { - throw new FileNotFoundException("Enterprise data file not found"); - } + this.config = new FiftyOneConfig(dataFile); - pipeline = new DeviceDetectionPipelineBuilder() + // The Lite data file has only some of the properties, so it is + // given none to restrict to, and the rest read as needing the + // Enterprise data file. + lite = "Lite".equals(DataFileHelper.getDatafileMetaData( + this.config.dataFile).getTier()); + DeviceDetectionOnPremisePipelineBuilder builder = + new DeviceDetectionPipelineBuilder() .useOnPremise(this.config.dataFile, false) .setPerformanceProfile(Constants.PerformanceProfiles.MaxPerformance) .setUsePerformanceGraph(true) .setUsePredictiveGraph(false) .setShareUsage(false) .setAutoUpdate(false) - .setConcurrency(numberOfThreads) - .setProperty("hardwareVendor") - .setProperty("hardwareModel") - .setProperty("browserVendor") - .setProperty("browserVersion") - .setProperty("isMobile") - .setProperty("deviceType") - .build(); + .setConcurrency(numberOfThreads); + if (lite == false) { + for (String property : PROPERTIES) { + builder.setProperty(property); + } + } + pipeline = builder.build(); + + DeviceDetectionHashEngine engine = + pipeline.getElement(DeviceDetectionHashEngine.class); + Set notInFile = new HashSet<>(); + for (String property : PROPERTIES) { + if (DataFileHelper.hasProperty(engine, property) == false) { + notInFile.add(property); + } + } + missing = notInFile; } @Override public Detection.Properties detect(Detection.Request request) throws Exception { try (FlowData flowData = pipeline.createFlowData()){ flowData.addEvidence(request.getEvidence()).process(); - return new FiftyOneProperties(request, flowData); + return new FiftyOneProperties(request, flowData, missing); + } + } + + /** + * On the free Lite data file, says what the Enterprise data file + * adds and where to ask for a copy. + */ + @Override + public void writeNote(PrintWriter writer) { + if (lite) { + DataFileHelper.writeEnterpriseOffer(writer, CONTACT_US); } } diff --git a/examples/device-detection/console/src/main/resources/gettingStartedOnPrem.xml b/examples/device-detection/console/src/main/resources/gettingStartedOnPrem.xml index ee634b786..520b105bb 100644 --- a/examples/device-detection/console/src/main/resources/gettingStartedOnPrem.xml +++ b/examples/device-detection/console/src/main/resources/gettingStartedOnPrem.xml @@ -14,4 +14,13 @@ JavaScriptBuilderElement --> + + + false + diff --git a/examples/device-detection/console/src/main/resources/tacCloud.xml b/examples/device-detection/console/src/main/resources/tacCloud.xml index e94963325..48f84bcc8 100644 --- a/examples/device-detection/console/src/main/resources/tacCloud.xml +++ b/examples/device-detection/console/src/main/resources/tacCloud.xml @@ -30,12 +30,12 @@ from our pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-resources-taccloud.xml&utm_term=buildparameters. Once this is done, a resource key with the properties required by this example - can be created at https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-resources-taccloud.xml&utm_term=buildparameters. + can be created at https://51degrees.com/configure/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-resources-taccloud.xml&utm_term=buildparameters. You can paste the resource key here, or set an environment - variable or System property called "SuperResourceKey" + variable or System property called "_51DEGREES_RESOURCE_KEY_SUPER" which will get inserted as the value.--> - ${SuperResourceKey} + ${_51DEGREES_RESOURCE_KEY_SUPER} CloudRequestEngine diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/CloudDropInTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/CloudDropInTest.java new file mode 100644 index 000000000..9d4eaabe4 --- /dev/null +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/CloudDropInTest.java @@ -0,0 +1,152 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.console; + +import com.sun.net.httpserver.HttpServer; +import fiftyone.devicedetection.DeviceDetectionPipelineBuilder; +import fiftyone.devicedetection.shared.DeviceData; +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; + +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * A cloud engine is a drop-in replacement for an on-premise engine: the + * same code reads the results of either + * (device-detection-specification/usage-examples.md, "In particular, a + * cloud Engine MUST be a drop-in replacement for an on-premise Engine for + * performing Device Detection and querying the results", and "However, + * once created, usage SHOULD be as similar as possible"). + * + * {@link #isMobile(Pipeline)} is that code. It reads the result by its type, + * by the element data key "device", and a property by name, and it is given + * a cloud pipeline and an on-premise one. The cloud pipeline talks to a + * stand-in for the cloud service that this test starts on this machine, so + * no resource key or network is needed. The on-premise pipeline uses the + * free Lite data file, which a fresh clone fetches from the data + * repository; the test does not download it. + */ +public class CloudDropInTest { + + private static final String IPHONE = + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) " + + "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 " + + "Mobile/15E148 Safari/604.1"; + + private HttpServer cloud; + + @Before + public void startCloud() throws Exception { + cloud = HttpServer.create( + new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + cloud.createContext("/", exchange -> { + try (InputStream body = exchange.getRequestBody()) { + byte[] buffer = new byte[4096]; + while (body.read(buffer) >= 0) { + // Read the whole request before answering. + } + } + String path = exchange.getRequestURI().getPath(); + String answer; + if (path.endsWith("accessibleproperties")) { + answer = "{\"Products\":{\"device\":{\"DataTier\":\"Lite\"," + + "\"Properties\":[{\"Name\":\"IsMobile\"," + + "\"Type\":\"Boolean\",\"Category\":\"Device\"}]}}}"; + } else if (path.endsWith("evidencekeys")) { + answer = "[\"header.user-agent\"]"; + } else { + answer = "{\"device\":{\"ismobile\":true}," + + "\"javascriptProperties\":[]}"; + } + byte[] bytes = answer.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", + "application/json"); + exchange.sendResponseHeaders(200, bytes.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(bytes); + } + }); + cloud.start(); + } + + @After + public void stopCloud() { + cloud.stop(0); + } + + /** + * The code a developer writes once, for either engine. + */ + private static boolean isMobile(Pipeline pipeline) throws Exception { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", IPHONE).process(); + DeviceData device = data.get(DeviceData.class); + ElementData byKey = data.get("device"); + assertSame("The result read by type and by the key 'device' " + + "should be the same", device, byKey); + Object byName = byKey.get("ismobile"); + assertTrue("The property read by name should be a value " + + "with a reason when it has none, and was " + byName, + byName instanceof AspectPropertyValue); + return device.getIsMobile().getValue(); + } + } + + @Test + public void cloudResultsReadAsOnPremise() throws Exception { + String endPoint = "http://" + cloud.getAddress().getHostString() + + ":" + cloud.getAddress().getPort() + "/api/v4/"; + try (Pipeline pipeline = new DeviceDetectionPipelineBuilder() + .useCloud("dropinresourcekey") + .setEndPoint(endPoint) + .build()) { + assertTrue(isMobile(pipeline)); + } + } + + @Test + public void onPremiseResultsReadAsCloud() throws Exception { + String file = getFilePath("device-detection/51Degrees-LiteV4.1.hash") + .getAbsolutePath(); + try (Pipeline pipeline = new DeviceDetectionPipelineBuilder() + .useOnPremise(file, false) + .setAutoUpdate(false) + .setShareUsage(false) + .build()) { + assertTrue(isMobile(pipeline)); + } + } +} diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/CloudExamplesWithoutKeyTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/CloudExamplesWithoutKeyTest.java new file mode 100644 index 000000000..2f1603a73 --- /dev/null +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/CloudExamplesWithoutKeyTest.java @@ -0,0 +1,98 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.console; + +import fiftyone.devicedetection.examples.shared.ExampleProcess; +import fiftyone.devicedetection.examples.shared.FindOutMore; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Runs each cloud example as it runs from a fresh clone, with no resource + * key anywhere, and checks that it ends without an error, names the + * variable to set and links to where a key is created, then ends with its + * Find out more section. + */ +public class CloudExamplesWithoutKeyTest { + + private static final String CONFIGURE = "https://51degrees.com/configure/"; + + private static final String TAGS = + "?utm_source=code&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content="; + + private static final String CONTACT_US = + "https://51degrees.com/contact-us" + TAGS; + + private static String runWithoutKey(Class example, String variable) + throws Exception { + ExampleProcess.Result result = ExampleProcess.runWithoutKeys(example); + System.out.println(result.output); + String output = result.output; + assertEquals(example.getSimpleName() + " should end with exit code 0", + 0, result.exitCode); + assertTrue("The output should name " + variable, + output.contains("Set the " + variable + " environment variable")); + int link = output.indexOf(CONFIGURE); + assertTrue("The output should link to where a key is created", + link >= 0 && output.startsWith(TAGS, + output.indexOf('?', link))); + assertTrue("The output should end with the '" + FindOutMore.HEADING + + "' section after the link", + output.lastIndexOf(FindOutMore.HEADING) > link); + return output; + } + + @Test + public void gettingStartedCloud() throws Exception { + runWithoutKey(GettingStartedCloud.class, "_51DEGREES_RESOURCE_KEY"); + } + + @Test + public void metadataCloud() throws Exception { + runWithoutKey(MetadataCloud.class, "_51DEGREES_RESOURCE_KEY"); + } + + @Test + public void minimalExample() throws Exception { + runWithoutKey(MinimalExample.class, "_51DEGREES_RESOURCE_KEY"); + } + + @Test + public void tacCloud() throws Exception { + String output = runWithoutKey(TacCloud.class, + "_51DEGREES_RESOURCE_KEY_SUPER"); + assertTrue("The output should link to the contact page", + output.contains(CONTACT_US)); + } + + @Test + public void nativeModelCloud() throws Exception { + String output = runWithoutKey(NativeModelCloud.class, + "_51DEGREES_RESOURCE_KEY_SUPER"); + assertTrue("The output should link to the contact page", + output.contains(CONTACT_US)); + } +} diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ConsoleConfigurationCoverageTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ConsoleConfigurationCoverageTest.java new file mode 100644 index 000000000..ffa2e69cd --- /dev/null +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ConsoleConfigurationCoverageTest.java @@ -0,0 +1,81 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.console; + +import fiftyone.pipeline.core.configuration.ElementOptions; +import fiftyone.pipeline.core.configuration.PipelineOptions; +import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; +import org.junit.Test; + +import java.io.File; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * The console examples show both ways of making a pipeline, for both + * on-premise and cloud: the fluent builder (GettingStartedCloud and the + * other on-premise consoles, run by their own tests) and an options file, + * checked here without a data file or a key, as the tests that run the + * examples need one or the other + * (device-detection-specification/required-examples.md, "Examples MUST be + * provided that illustrate basic usage of the API for both Cloud and + * On-Prem, using fluent builder and options file configuration, when + * executed as a console (command line) application"). + */ +public class ConsoleConfigurationCoverageTest { + + /** + * The builders an options file names, reading the file this example + * module holds in its resources, so that a file of the same name in + * another example cannot be read in its place. + */ + private static List builders(String file) throws Exception { + URL resource = + ConsoleConfigurationCoverageTest.class.getResource("/" + file); + assertNotNull("The example's resources hold no " + file, resource); + PipelineOptions options = PipelineOptionsFactory.getOptionsFromFile( + new File(resource.toURI())); + List names = new ArrayList<>(); + for (ElementOptions element : options.elements) { + names.add(element.builderName); + } + return names; + } + + @Test + public void onPremiseOptionsFile() throws Exception { + List builders = builders("gettingStartedOnPrem.xml"); + assertTrue(builders.toString(), + builders.contains("DeviceDetectionHashEngine")); + } + + @Test + public void cloudOptionsFile() throws Exception { + List builders = builders("tacCloud.xml"); + assertTrue(builders.toString(), builders.contains("CloudRequestEngine")); + } +} diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ExamplesWithoutSettingsTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ExamplesWithoutSettingsTest.java index aea911082..7fae33f00 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ExamplesWithoutSettingsTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ExamplesWithoutSettingsTest.java @@ -22,7 +22,11 @@ package fiftyone.devicedetection.examples.console; +import fiftyone.devicedetection.examples.console.comparison.Comparer; +import fiftyone.devicedetection.examples.console.comparison.DetectionImplBrowsCap; +import fiftyone.devicedetection.examples.console.comparison.DetectionImplFiftyOneDegrees; import fiftyone.devicedetection.examples.shared.DataFileHelper; +import fiftyone.devicedetection.examples.shared.FindOutMore; import org.junit.Test; import java.io.ByteArrayOutputStream; @@ -75,6 +79,7 @@ private static String runWithoutSettings(Main main) throws Exception { } String output = buffer.toString("UTF-8"); out.println(output); + assertFindOutMore(output); return output; } @@ -83,6 +88,21 @@ private static void assertContains(String output, String expected) { output.contains(expected)); } + /** + * Checks that the output has the "Find out more" section after the last + * link to the contact page, and that the section closes with its rule. + * Log lines may still follow the section. + */ + private static void assertFindOutMore(String output) { + int heading = output.lastIndexOf(FindOutMore.HEADING); + assertTrue("The output should have the '" + FindOutMore.HEADING + + "' section after the link to the contact page", + heading > output.lastIndexOf(CONTACT_US)); + assertTrue("The '" + FindOutMore.HEADING + + "' section should end with its rule", + output.indexOf(FindOutMore.RULE, heading) > heading); + } + private static void assertOffer(String output) { assertContains(output, "This example used the free Lite data file."); assertContains(output, CONTACT_US); @@ -119,6 +139,14 @@ public void performance() throws Exception { assertOffer(runWithoutSettings(PerformanceBenchmark::main)); } + @Test + public void manualDataUpdate() throws Exception { + String output = runWithoutSettings(ManualDataUpdate::main); + assertContains(output, "Hardware Vendor: " + DataFileHelper.NEEDS_ENTERPRISE); + assertContains(output, "The Enterprise data file adds daily automatic updates."); + assertContains(output, CONTACT_US); + } + @Test public void updateDataFile() throws Exception { String output = runWithoutSettings(UpdateDataFile::main); @@ -126,4 +154,18 @@ public void updateDataFile() throws Exception { assertContains(output, "which needs a licence key"); assertContains(output, CONTACT_US); } + + /** + * The comparison runs both solutions to the end on the free Lite data + * file and then says what the Enterprise data file adds. + */ + @Test + public void comparer() throws Exception { + String output = runWithoutSettings(Comparer::main); + assertContains(output, "Vendor: " + + DetectionImplFiftyOneDegrees.FIFTY_ONE_DEGREES + ","); + assertContains(output, "Vendor: " + + DetectionImplBrowsCap.BROWS_CAP_BLUECONIC + ","); + assertOffer(output); + } } diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/FindOutMoreCoverageTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/FindOutMoreCoverageTest.java new file mode 100644 index 000000000..f090ac8fe --- /dev/null +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/FindOutMoreCoverageTest.java @@ -0,0 +1,74 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.console; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertTrue; + +/** + * Checks that every console example with a main method writes the "Find out + * more" section, including those that need a key or a paid data file and so + * do not run in ExamplesWithoutSettingsTest. + */ +public class FindOutMoreCoverageTest { + + @Test + public void everyExample_WritesFindOutMore() throws IOException { + Path sources = Paths.get(System.getProperty("basedir", "."), + "src", "main", "java"); + assertTrue(sources + " should hold the examples", + Files.isDirectory(sources)); + List files; + try (Stream walk = Files.walk(sources)) { + files = walk.filter(f -> f.toString().endsWith(".java")) + .collect(Collectors.toList()); + } + List examples = new ArrayList<>(); + List missing = new ArrayList<>(); + for (Path file : files) { + String text = new String(Files.readAllBytes(file), + StandardCharsets.UTF_8); + if (text.contains("static void main(")) { + examples.add(file.getFileName().toString()); + if (text.contains("FindOutMore.write") == false) { + missing.add(file.getFileName().toString()); + } + } + } + assertTrue("No examples were found in " + sources, + examples.isEmpty() == false); + assertTrue("These examples do not write the 'Find out more' " + + "section: " + missing, missing.isEmpty()); + } +} diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java index 779a70fc2..7ceb26b14 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremTest.java @@ -22,17 +22,15 @@ package fiftyone.devicedetection.examples.console; +import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.devicedetection.examples.shared.EvidenceHelper; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; import org.junit.Test; public class GettingStartedOnPremTest { @Test public void gettingStartedOnPremTest() throws Exception { - GettingStartedOnPrem.run(FileUtils.getHashFileName() == null - ? FileUtils.LITE_HASH_DATA_FILE_PATH - : FileUtils.getHashFileName(), + GettingStartedOnPrem.run(DataFileHelper.getDataFileLocation(null), EvidenceHelper.setUpEvidence(), System.out); } } \ No newline at end of file diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremUsageSharingTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremUsageSharingTest.java new file mode 100644 index 000000000..d7eb7662d --- /dev/null +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedOnPremUsageSharingTest.java @@ -0,0 +1,93 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.console; + +import fiftyone.pipeline.core.configuration.PipelineOptions; +import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.fiftyone.flowelements.FiftyOnePipelineBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.ShareUsageElement; +import org.junit.Test; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; + +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * The on-premise getting started console example does not share usage + * (device-detection-specification/required-examples.md, "Reference + * implementations inhibit usage sharing for console examples by ensuring the + * Usage Sharing Element is not added to the Pipeline, but enable it for web + * examples", and "Every example in which usage sharing is inhibited needs to + * have a comment saying that in normal operation usage sharing should be + * enabled by adding a Usage Sharing Element to the Pipeline"). + * + * The example builds its Pipeline from its options file with the + * FiftyOnePipelineBuilder, which adds a Usage Sharing Element unless told + * not to. The test builds the Pipeline from the same file in the same way, + * without the engine, which needs a data file, and checks that no Usage + * Sharing Element is added. Building shares nothing; only processing does. + */ +public class GettingStartedOnPremUsageSharingTest { + + private static final String OPTIONS = + "examples/device-detection/console/src/main/resources/gettingStartedOnPrem.xml"; + + private static final String SOURCE = + "examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java"; + + private static String read(String path) throws Exception { + File file = getFilePath(path); + return new String(Files.readAllBytes(file.toPath()), + StandardCharsets.UTF_8).replaceAll("\\s+", " "); + } + + @Test + public void pipelineHasNoUsageSharingElement() throws Exception { + PipelineOptions options = + PipelineOptionsFactory.getOptionsFromFile(getFilePath(OPTIONS)); + options.elements.clear(); + try (Pipeline pipeline = new FiftyOnePipelineBuilder() + .buildFromConfiguration(options)) { + for (FlowElement element : pipeline.getFlowElements()) { + assertFalse("The example's Pipeline should not share usage", + element instanceof ShareUsageElement); + } + } + } + + @Test + public void commentSaysToShareUsageInProduction() throws Exception { + String expected = "in normal operation usage sharing should be " + + "enabled by adding a Usage Sharing Element to the Pipeline"; + assertTrue("The options file should say " + expected, + read(OPTIONS).contains(expected)); + assertTrue("The example should say " + expected, + read(SOURCE).contains(expected)); + } +} diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ManualDataUpdateTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ManualDataUpdateTest.java new file mode 100644 index 000000000..b33d91b5a --- /dev/null +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ManualDataUpdateTest.java @@ -0,0 +1,68 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.console; + +import fiftyone.devicedetection.examples.shared.DataFileHelper; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Runs the example that reloads the data file when its code says so, + * rather than through an automatic update, and checks that it detects + * the same User-Agents before and after the reload. + */ +public class ManualDataUpdateTest { + + private static final String CHECK = "a mobile device?"; + + @Test + public void manualDataUpdate_ReloadsTheDataFile() throws Exception { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + ManualDataUpdate.run( + DataFileHelper.getDataFileLocation(null), buffer); + String output = buffer.toString("UTF-8"); + System.out.println(output); + + int reloading = output.indexOf("Reloading the data file"); + int reloaded = output.indexOf("Reloaded the data file"); + assertTrue("The example should say it reloads the data file", + reloading >= 0); + assertTrue("The example should say it reloaded the data file", + reloaded > reloading); + assertEquals(2, count(output.substring(0, reloading), CHECK)); + assertEquals(2, count(output.substring(reloaded), CHECK)); + } + + private static int count(String text, String part) { + int count = 0; + for (int i = text.indexOf(part); i >= 0; + i = text.indexOf(part, i + part.length())) { + count++; + } + return count; + } +} diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java index a128b1393..cc2fa6573 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataOnPremTest.java @@ -22,14 +22,13 @@ package fiftyone.devicedetection.examples.console; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; +import fiftyone.devicedetection.examples.shared.DataFileHelper; import org.junit.Test; public class MetadataOnPremTest { @Test public void gettingMetaDataOnPremTest() throws Exception { - MetadataOnPrem.run(FileUtils.getHashFileName() == null - ? FileUtils.LITE_HASH_DATA_FILE_PATH - : FileUtils.getHashFileName(), System.out); + MetadataOnPrem.run( + DataFileHelper.getDataFileLocation(null), System.out); } } \ No newline at end of file diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java index 4861f14d6..15d517f04 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/OfflineProcessingTest.java @@ -22,19 +22,16 @@ package fiftyone.devicedetection.examples.console; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; +import fiftyone.devicedetection.examples.shared.DataFileHelper; import org.junit.Test; import java.io.FileInputStream; -import java.util.Objects; public class OfflineProcessingTest { @Test public void offlineProcessingTest() throws Exception { - OfflineProcessing.run(FileUtils.getHashFileName() == null - ? FileUtils.LITE_HASH_DATA_FILE_PATH - : FileUtils.getHashFileName(), - new FileInputStream(Objects.requireNonNull(FileUtils.getEvidenceFile())), + OfflineProcessing.run(DataFileHelper.getDataFileLocation(null), + new FileInputStream(DataFileHelper.getEvidenceFile(null)), System.out); } diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ProgrammaticUpdateTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ProgrammaticUpdateTest.java new file mode 100644 index 000000000..b14f856da --- /dev/null +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ProgrammaticUpdateTest.java @@ -0,0 +1,95 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.console; + +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import fiftyone.pipeline.engines.services.DataUpdateService; +import org.junit.Test; + +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +/** + * The data update example illustrates a programmatic (non-automatic) + * update as well as update on start-up, the file system watcher and + * polling (device-detection-specification/required-examples.md, "The data + * update service has a range of options available. Implementors SHOULD + * illustrate:", the last of which is "programmatic (non-automatic) + * update"). + * + * An update needs a licence key and the Enterprise data file, so the test + * gives the example's update step a stand-in service and engine, and checks + * that the step asks the service to check for an update of that engine now + * and gives back the status the service answers. + */ +public class ProgrammaticUpdateTest { + + /** + * A stand-in for an interface that answers Object's methods and records + * every other call. + */ + @SuppressWarnings("unchecked") + private static T standIn(Class type, List calls, + Object answer) { + return (T) Proxy.newProxyInstance( + type.getClassLoader(), + new Class[]{type}, + (proxy, method, args) -> { + switch (method.getName()) { + case "hashCode": + return System.identityHashCode(proxy); + case "equals": + return proxy == args[0]; + case "toString": + return type.getSimpleName(); + default: + calls.add(new Object[]{method.getName(), args}); + return answer; + } + }); + } + + @Test + public void updateNow_AsksTheServiceToCheckThatEngineNow() { + List engineCalls = new ArrayList<>(); + List serviceCalls = new ArrayList<>(); + OnPremiseAspectEngine engine = + standIn(OnPremiseAspectEngine.class, engineCalls, null); + DataUpdateService service = standIn(DataUpdateService.class, + serviceCalls, + DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_NOT_NEEDED); + + DataUpdateService.AutoUpdateStatus status = + UpdateDataFile.updateNow(service, engine); + + assertEquals(DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_NOT_NEEDED, + status); + assertEquals(1, serviceCalls.size()); + assertEquals("checkForUpdate", serviceCalls.get(0)[0]); + assertSame(engine, ((Object[]) serviceCalls.get(0)[1])[0]); + } +} diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/RandomIpTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/RandomIpTest.java new file mode 100644 index 000000000..ab437c3c0 --- /dev/null +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/RandomIpTest.java @@ -0,0 +1,53 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.console; + +import org.junit.Test; + +import java.util.Locale; + +import static org.junit.Assert.assertTrue; + +/** + * The client IP address ShareUsageExample makes up for each record is + * Evidence that an engine reads, so it is written with the digits 0 to 9 + * whatever the default locale. + */ +public class RandomIpTest { + + private static final int ADDRESSES = 20; + + @Test + public void randomIp_ArabicDefaultLocale_AsciiDigits() { + Locale saved = Locale.getDefault(); + Locale.setDefault(Locale.forLanguageTag("ar-SA")); + try { + for (int i = 0; i < ADDRESSES; i++) { + String ip = ShareUsageExample.generateRandomIp(); + assertTrue(ip, ip.matches("[0-9]{1,3}(\\.[0-9]{1,3}){3}")); + } + } finally { + Locale.setDefault(saved); + } + } +} diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ShareUsageExampleTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ShareUsageExampleTest.java new file mode 100644 index 000000000..077feb77c --- /dev/null +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ShareUsageExampleTest.java @@ -0,0 +1,106 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.console; + +import com.sun.net.httpserver.HttpServer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertTrue; + +/** + * Runs the usage sharing example, so that a change that stops it working + * is seen. The specification asks for a unit test of every example + * (device-detection-specification/required-examples.md, "There MUST be unit + * tests for examples"). The example's messages go to an endpoint the test + * starts on this machine, so the test needs no network and shares nothing + * with 51Degrees. The evidence is held in the test, so no data file is + * needed either. + */ +public class ShareUsageExampleTest { + + private HttpServer server; + private final AtomicInteger messages = new AtomicInteger(); + + @Before + public void startEndpoint() throws Exception { + server = HttpServer.create( + new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + server.createContext("/", exchange -> { + try (InputStream body = exchange.getRequestBody()) { + byte[] buffer = new byte[4096]; + while (body.read(buffer) >= 0) { + // Read the whole message before answering. + } + } + messages.incrementAndGet(); + exchange.sendResponseHeaders(200, -1); + exchange.close(); + }); + server.start(); + } + + @After + public void stopEndpoint() { + server.stop(0); + } + + /** + * YAML documents in the shape of the 20,000 evidence records file, one + * set of evidence each, enough for more than one message at the + * example's batch size. + */ + private static InputStream evidence(int records) { + StringBuilder yaml = new StringBuilder(); + for (int i = 0; i < records; i++) { + yaml.append("---\n") + .append("header.user-agent: 'Mozilla/5.0 (Windows NT 10.0; ") + .append("Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ") + .append("Chrome/").append(100 + i).append(".0.0.0 Safari/537.36'\n") + .append("header.sec-ch-ua-mobile: '?0'\n") + .append("query.ignored: 'not a header, so not shared'\n"); + } + return new ByteArrayInputStream(yaml.toString() + .getBytes(StandardCharsets.UTF_8)); + } + + @Test + public void runsAndShares() throws Exception { + String url = "http://" + server.getAddress().getHostString() + ":" + + server.getAddress().getPort() + "/new.ashx"; + + ShareUsageExample.run(evidence(30), url); + + assertTrue("The example should send its evidence to the usage " + + "sharing endpoint it is given, and it sent nothing", + messages.get() > 0); + } +} diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/UpdateDataFileTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/UpdateDataFileTest.java index 301984e5a..cee7cc298 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/UpdateDataFileTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/UpdateDataFileTest.java @@ -22,7 +22,7 @@ package fiftyone.devicedetection.examples.console; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; +import fiftyone.devicedetection.examples.shared.KeyHelper; import org.junit.Test; import static fiftyone.devicedetection.examples.console.UpdateDataFile.run; @@ -33,9 +33,11 @@ public class UpdateDataFileTest { @Test public void updateExampleTest () throws Exception { - String licenseKey = KeyUtils.getNamedKey(UpdateDataFile.UPDATE_EXAMPLE_LICENSE_KEY_NAME); - assumeFalse("Skipping test, no license key found", - KeyUtils.isInvalidKey(licenseKey)); + String licenseKey = KeyHelper.getNamedKey(UpdateDataFile.UPDATE_EXAMPLE_LICENSE_KEY_NAME); + assumeFalse("Skipped, because " + + UpdateDataFile.UPDATE_EXAMPLE_LICENSE_KEY_NAME + + " is not set to a license key for the data files.", + KeyHelper.isInvalidKey(licenseKey)); try { run(null, licenseKey, false); } catch (IllegalStateException e) { diff --git a/examples/device-detection/pom.xml b/examples/device-detection/pom.xml index 8e9dd78bc..946493ee1 100644 --- a/examples/device-detection/pom.xml +++ b/examples/device-detection/pom.xml @@ -62,7 +62,7 @@ 2.0.20 1.3.16 - 3.26.0 + 3.28.0 @@ -149,13 +149,12 @@ ${maven.multiModuleProjectDirectory} - - ${TestResourceKey} - - ${SuperResourceKey} - - ${LicenseKey} + + <_51DEGREES_RESOURCE_KEY>${_51DEGREES_RESOURCE_KEY} + <_51DEGREES_RESOURCE_KEY_SUPER>${_51DEGREES_RESOURCE_KEY_SUPER} + <_51DEGREES_LICENSE_KEY>${_51DEGREES_LICENSE_KEY} diff --git a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java index 55a9c44c1..e2acf185d 100644 --- a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java +++ b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java @@ -24,7 +24,6 @@ import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngineBuilder; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; import fiftyone.pipeline.engines.data.AspectEngineDataFile; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneDataFile; @@ -75,6 +74,24 @@ public class DataFileHelper { */ public static final String NEEDS_ENTERPRISE = "needs the Enterprise data file"; + public static final String TAC_HASH_DATA_FILE_NAME = "TAC-HashV41.hash"; + + public static final String ENTERPRISE_HASH_DATA_FILE_NAME = "Enterprise-HashV41.hash"; + + /** + * Where the free Lite data file is in the data repository. A search for + * the path rather than the name does not stop at another file of the + * same name, such as a Git LFS pointer in an old device-detection-data + * folder. + */ + public static final String LITE_HASH_DATA_FILE_PATH = + "device-detection/51Degrees-LiteV4.1.hash"; + + /** + * Where the evidence the examples process is in the data repository. + */ + public static final String EVIDENCE_FILE_PATH = + "device-detection/20000 Evidence Records.yml"; public static class DatafileInfo { FiftyOneDataFile fileInfo; String tier; @@ -210,7 +227,7 @@ public static void writeEnterpriseOffer(PrintWriter output, String contactUs) { @SuppressWarnings("RedundantThrows") public static File getEvidenceFile(String evidenceFilename) throws Exception { if (Objects.isNull(evidenceFilename)) { - evidenceFilename = FileUtils.EVIDENCE_FILE_PATH; + evidenceFilename = EVIDENCE_FILE_PATH; } File evidenceFile; @@ -233,10 +250,10 @@ public static File getEvidenceFile(String evidenceFilename) throws Exception { @SuppressWarnings("RedundantThrows") public static String getDataFileLocation(String dataFilename) throws Exception { if (Objects.isNull(dataFilename)) { - dataFilename = FileUtils.getHashFileName(); + dataFilename = findHashFileName(); } if (Objects.isNull(dataFilename)) { - dataFilename = FileUtils.LITE_HASH_DATA_FILE_PATH; + dataFilename = LITE_HASH_DATA_FILE_PATH; } String dataFileLocation; try { @@ -248,6 +265,25 @@ public static String getDataFileLocation(String dataFilename) throws Exception { return dataFileLocation; } + /** + * The first Hash data file found below the project root, being the TAC + * file, then the Enterprise file, then the free Lite file. + * @return the path of the file, or null when there is none + */ + private static String findHashFileName() { + for (String file : new String[]{ + TAC_HASH_DATA_FILE_NAME, + ENTERPRISE_HASH_DATA_FILE_NAME, + LITE_HASH_DATA_FILE_PATH}) { + try { + return getFilePath(file).getPath(); + } catch (Exception e) { + // Not found, so try the next file. + } + } + return null; + } + /** * Finds a file below the project root, as * {@link FileFinder#getFilePath(String)} does, and then in the data diff --git a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/ExampleProcess.java b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/ExampleProcess.java new file mode 100644 index 000000000..b010dde43 --- /dev/null +++ b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/ExampleProcess.java @@ -0,0 +1,127 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.shared; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +/** + * Runs an example's main method in a new Java process with no key in its + * environment, as someone runs it from a fresh clone, for the tests of + * what an example does with no settings. A new process is needed because + * a running Java process cannot remove a variable from its own + * environment. + */ +public final class ExampleProcess { + + /** + * How long an example may run before the test stops it. + */ + private static final long TIMEOUT_SECONDS = 120; + + /** + * What an example wrote and how it ended. + */ + public static final class Result { + /** + * The process's exit code. + */ + public final int exitCode; + + /** + * Everything the process wrote to standard output and standard + * error. + */ + public final String output; + + Result(int exitCode, String output) { + this.exitCode = exitCode; + this.output = output; + } + } + + private ExampleProcess() { + } + + /** + * Runs the example's main method with no arguments and with every + * environment variable whose name holds "KEY", in any case, left out. + * The process takes the class path of the process that starts it, and + * of its system properties only project.root, which says where the + * files the example reads are. + * @param example the class whose main method runs + * @return the exit code and the output + * @throws Exception if the process cannot start, or does not end + * within the timeout + */ + public static Result runWithoutKeys(Class example) throws Exception { + List command = new ArrayList<>(); + command.add(new File(new File(System.getProperty("java.home"), + "bin"), "java").getPath()); + command.add("-cp"); + command.add(System.getProperty("java.class.path")); + String projectRoot = System.getProperty("project.root"); + if (projectRoot != null) { + command.add("-Dproject.root=" + projectRoot); + } + command.add(example.getName()); + + ProcessBuilder builder = new ProcessBuilder(command); + builder.environment().keySet().removeIf( + name -> name.toUpperCase(Locale.ROOT).contains("KEY")); + builder.redirectErrorStream(true); + Process process = builder.start(); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + Thread reader = new Thread(() -> copy(process.getInputStream(), + output)); + reader.start(); + if (process.waitFor(TIMEOUT_SECONDS, TimeUnit.SECONDS) == false) { + process.destroyForcibly(); + reader.join(); + throw new AssertionError(example.getName() + + " did not end within " + TIMEOUT_SECONDS + " seconds. " + + "It wrote:\n" + output.toString("UTF-8")); + } + reader.join(); + return new Result(process.exitValue(), output.toString("UTF-8")); + } + + private static void copy(InputStream input, ByteArrayOutputStream output) { + byte[] buffer = new byte[8192]; + try { + int read; + while ((read = input.read(buffer)) >= 0) { + output.write(buffer, 0, read); + } + } catch (IOException e) { + // The process ended, so there is no more to read. + } + } +} diff --git a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/FindOutMore.java b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/FindOutMore.java new file mode 100644 index 000000000..e50c93d35 --- /dev/null +++ b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/FindOutMore.java @@ -0,0 +1,200 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.shared; + +import java.io.OutputStream; +import java.io.PrintWriter; +import java.util.regex.Pattern; + +/** + * Writes the "Find out more" section that ends every example's output, set + * apart from the example's own output by a rule above and below. It gives + * pages on 51degrees.com, each tagged with the example that wrote it, then + * the repositories behind the example. + */ +public final class FindOutMore { + + /** + * The heading the section starts with. + */ + public static final String HEADING = "Find out more"; + + /** + * The rule above and below the section. + */ + public static final String RULE = + "----------------------------------------------------------------"; + + /** + * The repositories an on-premise example lists, being this SDK, the C + * engines it builds on and the free data files. + */ + public static final String[] ON_PREMISE_REPOSITORIES = { + "https://github.com/51Degrees/51degrees-java", + "https://github.com/51Degrees/51degrees-cxx", + "https://github.com/51Degrees/data" + }; + + /** + * The repositories a cloud example lists. + */ + public static final String[] CLOUD_REPOSITORIES = { + "https://github.com/51Degrees/51degrees-java" + }; + + /** + * Each page's label and its path on 51degrees.com. + */ + private static final String[][] PAGES = { + {"Device Detection", "device-detection"}, + {"Pricing", "pricing"}, + {"Documentation", "documentation/index.html"} + }; + + /** + * What a campaign tag value may hold. + */ + private static final Pattern SLUG = + Pattern.compile("[a-z0-9][a-z0-9._-]*"); + + private FindOutMore() { + } + + /** + * Writes the section. + * @param output where to write, flushed afterwards + * @param example the example's tag, a lower case slug such as + * "console-gettingstartedonprem.java" + * @param repositories the repositories to list, such as + * {@link #ON_PREMISE_REPOSITORIES} + */ + public static void write( + PrintWriter output, + String example, + String... repositories) { + output.println(); + writeLines(output, "", example, repositories); + } + + /** + * Writes the section with each line a comment, for output that has to + * stay valid in its own format, such as a YAML stream. + * @param output where to write, flushed afterwards + * @param comment what starts a comment line, such as "# " + * @param example the example's tag + * @param repositories the repositories to list + */ + public static void writeCommented( + PrintWriter output, + String comment, + String example, + String... repositories) { + writeLines(output, comment, example, repositories); + } + + private static void writeLines( + PrintWriter output, + String start, + String example, + String... repositories) { + output.println(start + RULE); + output.println(start + HEADING); + for (String[] page : PAGES) { + output.println(start + "\t" + page[0] + ": " + + link(page[1], example)); + } + for (String repository : repositories) { + output.println(start + "\t" + repository); + } + output.println(start + RULE); + output.flush(); + } + + /** + * Writes the section to a stream. See + * {@link #write(PrintWriter, String, String...)}. + */ + public static void write( + OutputStream output, + String example, + String... repositories) { + write(new PrintWriter(output), example, repositories); + } + + /** + * The section as HTML, for the end of a web example's page. + * @param example the example's tag + * @param repositories the repositories to list + * @return the markup + */ + public static String html(String example, String... repositories) { + StringBuilder html = new StringBuilder(); + html.append("

\n"); + html.append("

").append(HEADING).append("

\n"); + html.append(" \n
\n"); + return html.toString(); + } + + /** + * The address of a page on 51degrees.com, tagged with the example. + * @param page the path of the page + * @param example the example's tag + * @return the address + * @throws IllegalArgumentException if the tag is not a lower case slug + */ + public static String link(String page, String example) { + return link(page, example, "find-out-more"); + } + + /** + * The address of a page on 51degrees.com, tagged with the example and + * the part of its output that gives the link. + * @param page the path of the page + * @param example the example's tag + * @param term the part of the output, a lower case slug + * @return the address + * @throws IllegalArgumentException if a tag is not a lower case slug + */ + public static String link(String page, String example, String term) { + for (String tag : new String[]{example, term}) { + if (tag == null || SLUG.matcher(tag).matches() == false) { + throw new IllegalArgumentException( + "'" + tag + "' is not a lower case slug."); + } + } + return "https://51degrees.com/" + + page + "?utm_source=code&utm_medium=example" + + "&utm_campaign=51degrees-java&utm_content=" + example + + "&utm_term=" + term; + } +} diff --git a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java index 467a443be..2d0c416a4 100644 --- a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java +++ b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java @@ -22,34 +22,149 @@ package fiftyone.devicedetection.examples.shared; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.PrintStream; import java.util.*; import java.util.function.Function; public class KeyHelper { - public static final String TEST_RESOURCE_KEY = "TestResourceKey"; + /** + * The variable the cloud examples and their tests read a resource key + * from. + */ + public static final String TEST_RESOURCE_KEY = "_51DEGREES_RESOURCE_KEY"; + + /** + * The variable the TAC and native model examples and their tests read + * the resource key with every property from. + */ + public static final String SUPER_RESOURCE_KEY_VARIABLE = + "_51DEGREES_RESOURCE_KEY_SUPER"; static Logger logger = LoggerFactory.getLogger(KeyHelper.class); /** - * The names every cloud test reads a resource key from first, before - * the names its own example reads. + * The shortest a key can be, as text and once decoded. + */ + private static final int MIN_KEY_LENGTH = 19; + + private static final int MIN_DECODED_KEY_LENGTH = 14; + + /** + * Read a key as an environment variable, then as its upper case name, + * then as a system property. + * @param keyName the name of the key + * @return the key, or null when none is set + */ + public static String getNamedKey(String keyName) { + String key = System.getenv(keyName); + if (Objects.isNull(key)) { + key = System.getenv(keyName.toUpperCase(Locale.ROOT)); + if (Objects.isNull(key)) { + key = System.getProperty(keyName); + } + } + return key; + } + + /** + * Whether a value cannot be a key, being missing, too short, or not URL + * safe base 64 of a long enough key. + * @param keyValue the value to check + * @return true if the value cannot be a key */ - public static final List SHARED_RESOURCE_KEY_NAMES = - Collections.unmodifiableList(Arrays.asList( - "_51DEGREES_RESOURCE_KEY", - "_51DEGREES_RESOURCE_KEY_BESPOKE")); + public static boolean isInvalidKey(String keyValue) { + try { + return Objects.isNull(keyValue) || + keyValue.trim().length() < MIN_KEY_LENGTH || + Base64.getUrlDecoder().decode(keyValue).length < + MIN_DECODED_KEY_LENGTH; + } catch (IllegalArgumentException e) { + return true; + } + } static final String CREATE_KEY_URL = - "https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-keyhelper.java&utm_term=no-resource-key"; + "https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-keyhelper.java&utm_term=no-resource-key"; + + /** + * The variable a cloud example reads its resource key from first. + */ + public static final String RESOURCE_KEY_VARIABLE = + "_51DEGREES_RESOURCE_KEY"; + + /** + * The configuration page that creates a key with the free properties + * the cloud examples use. + */ + public static final String FREE_PROPERTIES_CONFIGURATION = + "configure/jqz435Nc"; + + /** + * The configuration page that creates a key with the properties the + * TAC and native model examples use, which need a paid licence. + */ + public static final String PAID_PROPERTIES_CONFIGURATION = + "configure/QKyYH5XT"; + + /** + * The resource key a cloud example runs with, being the key given as + * its argument, or else the first of the names that holds one, read as + * an environment variable, under its upper case name and as a system + * property. + * @param argument the key given on the command line, or null + * @param names the names to read, in order + * @return the key, or null when there is none + */ + public static String findResourceKey(String argument, String... names) { + if (isInvalidKey(argument) == false) { + return argument.trim(); + } + for (String name : names) { + String value = readKey(name); + if (value != null) { + return value.trim(); + } + } + return null; + } + + /** + * Writes what a cloud example needs when it has no resource key, being + * the variable to set and the page where a key is created, so that the + * example can end without an error. + * @param output where to write + * @param example the example's tag, as its Find out more section has it + * @param variable the variable the example reads its key from + * @param configuration the configuration page for the key, such as + * {@link #FREE_PROPERTIES_CONFIGURATION} + */ + public static void writeNoResourceKey(PrintStream output, String example, + String variable, + String configuration) { + output.println(); + output.println("This example calls the 51Degrees cloud service, " + + "which needs a resource key, and none is set."); + output.println("Set the " + variable + " environment variable to a " + + "resource key, or give the key as the first argument, and run " + + "the example again."); + output.println("Create a resource key with the properties this " + + "example uses at " + + FindOutMore.link(configuration, example, "no-resource-key")); + if (PAID_PROPERTIES_CONFIGURATION.equals(configuration)) { + output.println("These properties need a paid licence. Contact " + + "us about one at " + + FindOutMore.link("contact-us", example, "no-resource-key")); + } + output.flush(); + } /** * The resource key for a test that calls the live cloud service, from - * {@link #SHARED_RESOURCE_KEY_NAMES} and then {@link #TEST_RESOURCE_KEY}. - * Fails the test when there is none, because a cloud test that passes - * without a key has called nothing. + * {@link #TEST_RESOURCE_KEY}. Skips the test, naming the variable, when + * there is none, because a cloud test that passes without a key has + * called nothing. * @param asksFor what the test asks the service for * @param products the products the key has to carry * @return the resource key @@ -62,7 +177,7 @@ public static String requireTestResourceKey(String asksFor, /** * The resource key for a test that calls the live cloud service, from - * {@link #SHARED_RESOURCE_KEY_NAMES} and then the given names. + * the given names only. Skips the test, naming them, when there is none. * @param exampleNames the names the example reads, in the order it * reads them * @param asksFor what the test asks the service for @@ -80,24 +195,43 @@ static String requireResourceKey(Function read, List exampleNames, String asksFor, String... products) { - List names = new ArrayList<>(SHARED_RESOURCE_KEY_NAMES); - names.addAll(exampleNames); - for (String name : names) { + for (String name : exampleNames) { String value = read.apply(name); - if (KeyUtils.isInvalidKey(value) == false) { + if (isInvalidKey(value) == false) { return value.trim(); } } - throw new AssertionError(noResourceKey(names, asksFor, products)); + throw skipTest(noResourceKey(exampleNames, asksFor, products)); + } + + /** + * The exception that skips a test, being JUnit's + * AssumptionViolatedException. It is made by name, because the + * examples use KeyHelper too, and run with no test library on their + * class path. Where JUnit cannot be found, the exception is an + * IllegalStateException with the same message. + * @param message why the test is skipped + * @return the exception to throw + */ + static RuntimeException skipTest(String message) { + try { + return (RuntimeException) Class.forName( + "org.junit.AssumptionViolatedException") + .getConstructor(String.class) + .newInstance(message); + } catch (ReflectiveOperationException | LinkageError | + ClassCastException e) { + return new IllegalStateException(message); + } } static String noResourceKey(List names, String asksFor, String... products) { return "This test calls the live 51Degrees cloud service, so it " + - "needs a resource key and there is nothing to fall back on. " + - "None of " + String.join(", ", names) + " is set, as an " + - "environment variable or as a system property, tried in that " + - "order. The key has to carry the " + productList(products) + + "needs a resource key, and it is skipped because " + + String.join(", ", names) + " is not set as an environment " + + "variable or as a system property. " + + "The key has to carry the " + productList(products) + ", because the test asks the service for " + asksFor + ". To " + "see the products a key carries, open " + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=. " + @@ -139,9 +273,9 @@ private static String productList(String... products) { private static String readKey(String name) { for (String value : new String[]{ System.getenv(name), - System.getenv(name.toUpperCase()), + System.getenv(name.toUpperCase(Locale.ROOT)), System.getProperty(name)}) { - if (KeyUtils.isInvalidKey(value) == false) { + if (isInvalidKey(value) == false) { return value; } } @@ -163,7 +297,7 @@ public static String getOrSetTestResourceKey(String value, boolean shouldThrow) return getOrSetResourceKey(value, TEST_RESOURCE_KEY, "A free resource key configured with the " + "properties required by this example may be obtained from " + - "https://configure.51degrees.com/jqz435Nc?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-keyhelper.java&utm_term=resource-key-required ", + "https://51degrees.com/configure/jqz435Nc?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-keyhelper.java&utm_term=resource-key-required ", shouldThrow); } public static String getOrSetTestResourceKey(String value) { @@ -178,10 +312,10 @@ public static String getOrSetResourceKey(String value, String variableName, String errorMessage, boolean shouldThrow) { if (Objects.isNull(value)) { - value = KeyUtils.getNamedKey(variableName); + value = getNamedKey(variableName); } - if (KeyUtils.isInvalidKey(value)) { + if (isInvalidKey(value)) { logger.error("\nTo access Cloud Services you must supply a " + "\"ResourceKey\" in one of the following ways: \n - in the " + "configuration file of an example,\n - as a command line parameter of a " + @@ -196,7 +330,7 @@ public static String getOrSetResourceKey(String value, String variableName, // Keep a valid key for next time. System.setProperty throws on null, // and an invalid key is not worth keeping. - if (KeyUtils.isInvalidKey(value) == false) { + if (isInvalidKey(value) == false) { System.setProperty(variableName, value); } return value; @@ -210,7 +344,7 @@ public static String getOrSetSuperResourceKey(String value, String variablename, "that you will first need a license key, which can be purchased " + "from our pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-keyhelper.java&utm_term=license-key-required. \nOnce this is " + "done, a resource key with the properties required by this example " + - "can be created at https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-keyhelper.java&utm_term=license-key-required. ", + "can be created at https://51degrees.com/configure/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-keyhelper.java&utm_term=license-key-required. ", shouldThrow); } } diff --git a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/PropertyHelper.java b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/PropertyHelper.java index 93da45886..f54e3e58f 100644 --- a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/PropertyHelper.java +++ b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/PropertyHelper.java @@ -47,8 +47,9 @@ public static AspectPropertyValue tryGet(Supplier> } catch (PropertyMissingException e) { String message = "The property '" + e.getPropertyName() + "' is not " + - "available in this data file. See data file options " + - "here"; + "available with the data file or resource key in use. " + + "Contact us about one that has it at " + + "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-propertyhelper.java&utm_term=property-not-available"; AspectPropertyValue result = new AspectPropertyValueDefault<>(); result.setNoValueMessage(message); return result; diff --git a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/settings.xml b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/settings.xml index cbbbbc9af..9ad03bccc 100644 --- a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/settings.xml +++ b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/settings.xml @@ -45,8 +45,8 @@ your IDE launch configuration or your operating system shell. - + https://51degrees.com/configure/jqz435Nc?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-settings.xml&utm_term=properties --> + <_51DEGREES_RESOURCE_KEY> diff --git a/examples/device-detection/shared/src/test/java/fiftyone/devicedetection/examples/shared/FindOutMoreTest.java b/examples/device-detection/shared/src/test/java/fiftyone/devicedetection/examples/shared/FindOutMoreTest.java new file mode 100644 index 000000000..8ff97350a --- /dev/null +++ b/examples/device-detection/shared/src/test/java/fiftyone/devicedetection/examples/shared/FindOutMoreTest.java @@ -0,0 +1,149 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.shared; + +import org.junit.Test; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class FindOutMoreTest { + + /** + * The campaign tags common-ci's scripts/utm-lint.ps1 requires of a link + * printed by an example. + */ + private static final Pattern TAGGED = Pattern.compile( + "https://51degrees\\.com/[^?\\s]+\\?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=[a-z0-9][a-z0-9._-]*" + + "&utm_term=[a-z0-9][a-z0-9._-]*"); + + private static String[] write(String example, String... repositories) { + StringWriter text = new StringWriter(); + FindOutMore.write(new PrintWriter(text), example, repositories); + return text.toString().split("\r?\n"); + } + + /** + * The section is a blank line, a rule, the heading, the three tagged + * pages, the repositories and a closing rule. + */ + @Test + public void write_LinksThenRepositories() { + String[] lines = write("console-example.java", + FindOutMore.ON_PREMISE_REPOSITORIES); + assertEquals(10, lines.length); + assertEquals("", lines[0]); + assertEquals(FindOutMore.RULE, lines[1]); + assertEquals(FindOutMore.HEADING, lines[2]); + assertTrue(lines[3].startsWith("\tDevice Detection: https://51degrees.com/device-detection?")); + assertTrue(lines[4].startsWith("\tPricing: https://51degrees.com/pricing?")); + assertTrue(lines[5].startsWith("\tDocumentation: https://51degrees.com/documentation/index.html?")); + for (int i = 3; i <= 5; i++) { + String link = lines[i].substring(lines[i].indexOf("https://")); + assertTrue(link, TAGGED.matcher(link).matches()); + assertTrue(link, link.contains("&utm_content=console-example.java&")); + } + assertEquals("\thttps://github.com/51Degrees/51degrees-java", lines[6]); + assertEquals("\thttps://github.com/51Degrees/51degrees-cxx", lines[7]); + assertEquals("\thttps://github.com/51Degrees/data", lines[8]); + assertEquals(FindOutMore.RULE, lines[9]); + } + + /** + * A cloud example lists this repository alone. + */ + @Test + public void write_Cloud_ThisRepository() { + String[] lines = write("console-example.java", + FindOutMore.CLOUD_REPOSITORIES); + assertEquals(8, lines.length); + assertEquals("\thttps://github.com/51Degrees/51degrees-java", lines[6]); + } + + /** + * Each line of the commented form starts with the comment marker, and + * there is no blank line, so a YAML stream stays valid. + */ + @Test + public void writeCommented_EveryLineAComment() { + StringWriter text = new StringWriter(); + FindOutMore.writeCommented(new PrintWriter(text), "# ", + "console-example.java", FindOutMore.CLOUD_REPOSITORIES); + String[] lines = text.toString().split("\\r?\\n"); + assertEquals(7, lines.length); + for (String line : lines) { + assertTrue(line, line.startsWith("# ")); + } + assertEquals("# " + FindOutMore.HEADING, lines[1]); + } + + /** + * The HTML form has the heading, each tagged page as a link with its + * ampersands escaped, and each repository as a link. + */ + @Test + public void html_LinksThenRepositories() { + String html = FindOutMore.html("web-example.java", + FindOutMore.ON_PREMISE_REPOSITORIES); + assertTrue(html, html.contains("

" + FindOutMore.HEADING + "

")); + java.util.regex.Matcher href = + Pattern.compile("href=\"([^\"]+)\"").matcher(html); + int pages = 0; + int repositories = 0; + while (href.find()) { + String link = href.group(1); + if (link.startsWith("https://51degrees.com/")) { + assertTrue(link, link.contains("&")); + assertTrue(link, + TAGGED.matcher(link.replace("&", "&")).matches()); + pages++; + } else { + assertTrue(link, link.startsWith("https://github.com/51Degrees/")); + repositories++; + } + } + assertEquals(3, pages); + assertEquals(3, repositories); + } + + /** + * A tag that is not a lower case slug would fail the campaign tag + * convention, so it is refused. + */ + @Test + public void link_NotASlug_Refused() { + assertThrows(IllegalArgumentException.class, + () -> FindOutMore.link("pricing", "Console-Example.java")); + assertThrows(IllegalArgumentException.class, + () -> FindOutMore.link("pricing", "console example")); + assertThrows(IllegalArgumentException.class, + () -> FindOutMore.link("pricing", null)); + } +} diff --git a/examples/device-detection/shared/src/test/java/fiftyone/devicedetection/examples/shared/KeyHelperTest.java b/examples/device-detection/shared/src/test/java/fiftyone/devicedetection/examples/shared/KeyHelperTest.java index 1a75b45a0..d21cc393a 100644 --- a/examples/device-detection/shared/src/test/java/fiftyone/devicedetection/examples/shared/KeyHelperTest.java +++ b/examples/device-detection/shared/src/test/java/fiftyone/devicedetection/examples/shared/KeyHelperTest.java @@ -22,13 +22,17 @@ package fiftyone.devicedetection.examples.shared; +import org.junit.AssumptionViolatedException; import org.junit.Test; +import java.net.URL; +import java.net.URLClassLoader; import java.util.Collections; import java.util.HashMap; import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; @@ -65,56 +69,73 @@ public void getOrSetTestResourceKey_NoKey_StoresNothing() { } @Test - public void requireResourceKey_NoKey_FailsNamingEveryName() { - AssertionError error = assertThrows(AssertionError.class, () -> - KeyHelper.requireResourceKey(name -> null, - Collections.singletonList(KeyHelper.TEST_RESOURCE_KEY), - "whether a device is mobile", "device")); + public void testResourceKey_IsTheSharedName() { + assertEquals("_51DEGREES_RESOURCE_KEY", KeyHelper.TEST_RESOURCE_KEY); + } + + /** + * A test with no key is skipped, naming the variable to set, rather + * than failed. + */ + @Test + public void requireResourceKey_NoKey_SkipsNamingTheVariable() { + AssumptionViolatedException skipped = assertThrows( + AssumptionViolatedException.class, () -> + KeyHelper.requireResourceKey(name -> null, + Collections.singletonList(KeyHelper.TEST_RESOURCE_KEY), + "whether a device is mobile", "device")); assertEquals("This test calls the live 51Degrees cloud service, so " + - "it needs a resource key and there is nothing to fall back " + - "on. None of _51DEGREES_RESOURCE_KEY, " + - "_51DEGREES_RESOURCE_KEY_BESPOKE, TestResourceKey is set, as " + - "an environment variable or as a system property, tried in " + - "that order. The key has to carry the device product, " + - "because the test asks the service for whether a device is " + - "mobile. To see the products a key carries, open " + + "it needs a resource key, and it is skipped because " + + "_51DEGREES_RESOURCE_KEY is not set as an environment " + + "variable or as a system property. The key has to carry the " + + "device product, because the test asks the service for " + + "whether a device is mobile. To see the products a key " + + "carries, open " + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=. " + "Create a key at " + KeyHelper.CREATE_KEY_URL + ".", - error.getMessage()); + skipped.getMessage()); } @Test public void requireResourceKey_TwoProducts_NamesBoth() { - AssertionError error = assertThrows(AssertionError.class, () -> - KeyHelper.requireResourceKey(name -> null, - Collections.singletonList("SuperResourceKey"), - "the devices that match a TAC", "hardware", "device")); - assertTrue(error.getMessage(), error.getMessage().contains( - "None of _51DEGREES_RESOURCE_KEY, " + - "_51DEGREES_RESOURCE_KEY_BESPOKE, SuperResourceKey is set")); - assertTrue(error.getMessage(), error.getMessage().contains( + AssumptionViolatedException skipped = assertThrows( + AssumptionViolatedException.class, () -> + KeyHelper.requireResourceKey(name -> null, + Collections.singletonList("_51DEGREES_RESOURCE_KEY_SUPER"), + "the devices that match a TAC", "hardware", "device")); + assertTrue(skipped.getMessage(), skipped.getMessage().contains( + "skipped because _51DEGREES_RESOURCE_KEY_SUPER is not set")); + assertTrue(skipped.getMessage(), skipped.getMessage().contains( "carry the hardware and device products, because")); } /** - * A name holding something that is not a key, such as the empty - * string the example poms pass for a key that is not set, is passed - * over. + * Only the names the test gives are read. A key under another name, + * such as the one every cloud example reads, does not stand in for the + * key with every property. */ @Test - public void requireResourceKey_FirstValidNameWins() { + public void requireResourceKey_ReadsOnlyTheNamesGiven() { Map values = new HashMap<>(); - values.put("_51DEGREES_RESOURCE_KEY", ""); - values.put("_51DEGREES_RESOURCE_KEY_BESPOKE", KEY_A); - values.put(KeyHelper.TEST_RESOURCE_KEY, KEY_B); - assertEquals(KEY_A, KeyHelper.requireResourceKey(values::get, - Collections.singletonList(KeyHelper.TEST_RESOURCE_KEY), - "whether a device is mobile", "device")); + values.put(KeyHelper.TEST_RESOURCE_KEY, KEY_A); + assertThrows(AssumptionViolatedException.class, () -> + KeyHelper.requireResourceKey(values::get, + Collections.singletonList("_51DEGREES_RESOURCE_KEY_SUPER"), + "the devices that match a TAC", "hardware", "device")); } + /** + * A name holding something that is not a key, such as the empty + * string a pom passes for a key that is not set, is passed over. + */ @Test - public void requireResourceKey_ExampleNameAfterSharedNames() { + public void requireResourceKey_ValueThatIsNotAKey_IsPassedOver() { Map values = new HashMap<>(); + values.put(KeyHelper.TEST_RESOURCE_KEY, ""); + assertThrows(AssumptionViolatedException.class, () -> + KeyHelper.requireResourceKey(values::get, + Collections.singletonList(KeyHelper.TEST_RESOURCE_KEY), + "whether a device is mobile", "device")); values.put(KeyHelper.TEST_RESOURCE_KEY, KEY_B); assertEquals(KEY_B, KeyHelper.requireResourceKey(values::get, Collections.singletonList(KeyHelper.TEST_RESOURCE_KEY), @@ -133,4 +154,45 @@ public void answeredWithout_NoReason_SaysSo() { KeyHelper.answeredWithout("BrowserName", "BrowserName is a paid feature", "device")); } + + /** + * A cloud example runs as an application, with no test library on its + * class path, and reads its key with KeyHelper. So KeyHelper has to + * load, and find a key, where JUnit cannot be found. + */ + @Test + public void findResourceKey_NoJUnitOnTheClassPath_FindsTheKey() + throws Exception { + try (URLClassLoader loader = withoutJUnit()) { + Class helper = Class.forName(KeyHelper.class.getName(), true, + loader); + assertNotSame(KeyHelper.class, helper); + String key = (String) helper + .getMethod("findResourceKey", String.class, String[].class) + .invoke(null, KEY_A, new String[0]); + assertEquals(KEY_A, key); + } + } + + /** + * A class loader with KeyHelper's own class files, that cannot see + * JUnit. + */ + private static URLClassLoader withoutJUnit() { + String helpers = KeyHelper.class.getPackage().getName() + "."; + ClassLoader parent = new ClassLoader( + KeyHelper.class.getClassLoader()) { + @Override + protected Class loadClass(String name, boolean resolve) + throws ClassNotFoundException { + if (name.startsWith("org.junit.") || + name.startsWith(helpers)) { + throw new ClassNotFoundException(name); + } + return super.loadClass(name, resolve); + } + }; + return new URLClassLoader(new URL[]{KeyHelper.class + .getProtectionDomain().getCodeSource().getLocation()}, parent); + } } diff --git a/examples/device-detection/shared/src/test/java/fiftyone/devicedetection/examples/shared/PropertyHelperMissingPropertyTests.java b/examples/device-detection/shared/src/test/java/fiftyone/devicedetection/examples/shared/PropertyHelperMissingPropertyTests.java new file mode 100644 index 000000000..8ea742972 --- /dev/null +++ b/examples/device-detection/shared/src/test/java/fiftyone/devicedetection/examples/shared/PropertyHelperMissingPropertyTests.java @@ -0,0 +1,70 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.shared; + +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyReason; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * A Property the data file or resource key in use cannot give is answered + * with plain text that names it and gives the tagged contact link, because + * the examples print the answer to a console, where HTML is read as it is + * written. + */ +public class PropertyHelperMissingPropertyTests { + + private static final String CONTACT_LINK = + "https://51degrees.com/contact-us?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-propertyhelper.java"; + + private static AspectPropertyValue missing() { + return PropertyHelper.tryGet(() -> { + throw new PropertyMissingException( + MissingPropertyReason.DataFileUpgradeRequired, + "HardwareVendor", + "HardwareVendor is not in this data file."); + }); + } + + @Test + public void MissingProperty_IsPlainText() { + String message = missing().getNoValueMessage(); + assertFalse("The message holds HTML: " + message, + message.contains("<") || message.contains("&")); + } + + @Test + public void MissingProperty_NamesItAndGivesTheContactLink() { + String message = missing().getNoValueMessage(); + assertTrue("The message does not name the property: " + message, + message.contains("'HardwareVendor'")); + assertTrue("The message has no tagged contact link: " + message, + message.contains(CONTACT_LINK)); + } +} diff --git a/examples/device-detection/shared/src/test/resources/51Degrees-Cloud-Test.xml b/examples/device-detection/shared/src/test/resources/51Degrees-Cloud-Test.xml index 335efff11..d02f4f902 100644 --- a/examples/device-detection/shared/src/test/resources/51Degrees-Cloud-Test.xml +++ b/examples/device-detection/shared/src/test/resources/51Degrees-Cloud-Test.xml @@ -27,7 +27,7 @@ http://cloud.51degrees.com/api/v4 - ${TestResourceKey} + ${_51DEGREES_RESOURCE_KEY} CloudRequestEngine @@ -51,8 +51,9 @@ JavaScriptBuilderElement http diff --git a/examples/device-detection/web/getting-started.cloud/src/main/webapp/WEB-INF/html/example-description.html b/examples/device-detection/web/getting-started.cloud/src/main/webapp/WEB-INF/html/example-description.html index 3cb275a57..0634d5b7c 100644 --- a/examples/device-detection/web/getting-started.cloud/src/main/webapp/WEB-INF/html/example-description.html +++ b/examples/device-detection/web/getting-started.cloud/src/main/webapp/WEB-INF/html/example-description.html @@ -24,9 +24,17 @@ device detection in a simple Web Application servlet project.

In particular, it highlights

  1. Automatic handling of the 'Accept-CH' header, which is used - to request User-Agent Client Hints from the browser;
  2. -
  3. Client-side evidence collection in order to identify Apple device - models and properties such as screen size.
+ to request User-Agent Client Hints from the browser. Chromium browsers + then send the high entropy values, such as the platform version and the + device model, as headers with their next request to this server, which + presents them to the 51Degrees cloud service with the other evidence; +
  • Client-side evidence collection. The 51Degrees JavaScript reads the + high entropy values with navigator.userAgentData.getHighEntropyValues + and sends them back to this server, which presents them to the + 51Degrees cloud service, so the first page a browser asks for gets + them too. The same script identifies Apple device models and + properties such as screen size. Browsers give high entropy values + only to a secure (https) origin or localhost.
  • The examples GettingStartedWebOnPrem and GettingStartedWebCloud differ only in their use of on-premise and cloud detection. diff --git a/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/examples/web/DefaultPortTest.java b/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/examples/web/DefaultPortTest.java new file mode 100644 index 000000000..9285c4d7f --- /dev/null +++ b/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/examples/web/DefaultPortTest.java @@ -0,0 +1,39 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.web; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Checks that the example listens on port 5151 when the PORT environment + * variable does not choose another, as the readme says. It needs no + * resource key, so it runs where the page test is skipped. + */ +public class DefaultPortTest { + @Test + public void listensOn5151ByDefault() { + assertEquals(5151, GettingStartedWebCloud.DEFAULT_PORT); + } +} diff --git a/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/CloudOptionsFileTest.java b/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/CloudOptionsFileTest.java new file mode 100644 index 000000000..1fb4db330 --- /dev/null +++ b/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/CloudOptionsFileTest.java @@ -0,0 +1,56 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.web; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertTrue; + +/** + * Checks the options file the cloud web example runs with. + */ +public class CloudOptionsFileTest { + + /** + * With no endpoint set, the example reaches the cloud service over + * HTTPS. + */ + @Test + public void defaultEndPoint_Https() throws IOException { + String xml = new String(Files.readAllBytes(Paths.get( + System.getProperty("basedir", "."), "src", "main", "webapp", + "WEB-INF", "51Degrees-Cloud.xml")), StandardCharsets.UTF_8); + Matcher endPoint = Pattern.compile( + "\\$\\{[^:}]*:-([^}]*)\\}").matcher(xml); + assertTrue("51Degrees-Cloud.xml should give a default endpoint", + endPoint.find()); + assertTrue(endPoint.group(1), endPoint.group(1).startsWith("https://")); + } +} diff --git a/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/GettingStartedWebCloudTest.java b/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/GettingStartedWebCloudTest.java index 11fa1f57a..761f92e35 100644 --- a/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/GettingStartedWebCloudTest.java +++ b/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/GettingStartedWebCloudTest.java @@ -22,8 +22,10 @@ package fiftyone.devicedetection.web; +import fiftyone.devicedetection.examples.shared.FindOutMore; import fiftyone.devicedetection.examples.shared.KeyHelper; import fiftyone.devicedetection.examples.web.EmbedJetty; +import fiftyone.devicedetection.examples.web.GettingStartedWebCloud; import org.eclipse.jetty.server.Server; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -69,13 +71,16 @@ public static void startJetty() throws Exception { "the browser and platform of a known User-Agent", "device"); // The example's configuration file reads the key from here. System.setProperty(KeyHelper.TEST_RESOURCE_KEY, resourceKey); - SERVER = EmbedJetty.startWebApp(getFilePath(resourceBase).getAbsolutePath(), 8081); + // Listen on the example's default port, so the request below to + // port 5151 checks that the example listens there. + SERVER = EmbedJetty.startWebApp(getFilePath(resourceBase).getAbsolutePath(), + GettingStartedWebCloud.DEFAULT_PORT); } @Test public void testWebCloud() throws Exception { HttpURLConnection connection = - (HttpURLConnection) new URL("http://localhost:8081/").openConnection(); + (HttpURLConnection) new URL("http://localhost:5151/").openConnection(); connection.setRequestProperty("User-Agent", USER_AGENT); int code = connection.getResponseCode(); @@ -90,6 +95,9 @@ public void testWebCloud() throws Exception { body.contains("cloud device detection engine")); assertDetected(body, "Browser Name", "BrowserName", "Chrome"); assertDetected(body, "Platform Name", "PlatformName", "Windows"); + assertTrue("The page should end with the '" + FindOutMore.HEADING + + "' section. " + body, body.contains( + "

    " + FindOutMore.HEADING + "

    ")); } /** diff --git a/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/GettingStartedWebCloudWithoutKeyTest.java b/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/GettingStartedWebCloudWithoutKeyTest.java new file mode 100644 index 000000000..3e509caac --- /dev/null +++ b/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/GettingStartedWebCloudWithoutKeyTest.java @@ -0,0 +1,68 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.web; + +import fiftyone.devicedetection.examples.shared.ExampleProcess; +import fiftyone.devicedetection.examples.web.GettingStartedWebCloud; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Runs the Device Detection cloud web example as it runs from a fresh + * clone, with no resource key anywhere, and checks that it ends without an + * error, names the variable to set and links to where a key is created, + * then ends with its Find out more section. + */ +public class GettingStartedWebCloudWithoutKeyTest { + + private static final String CONFIGURE = "https://51degrees.com/configure/"; + + private static final String TAGS = + "?utm_source=code&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content="; + + private static void runWithoutKey(Class example) throws Exception { + ExampleProcess.Result result = ExampleProcess.runWithoutKeys(example); + System.out.println(result.output); + String output = result.output; + assertEquals(example.getSimpleName() + " should end with exit code 0", + 0, result.exitCode); + assertTrue("The output should name _51DEGREES_RESOURCE_KEY", + output.contains("Set the _51DEGREES_RESOURCE_KEY " + + "environment variable")); + int link = output.indexOf(CONFIGURE); + assertTrue("The output should link to where a key is created", + link >= 0 && output.startsWith(TAGS, + output.indexOf('?', link))); + assertTrue("The output should end with the Find out more section " + + "after the link", + output.lastIndexOf("Find out more") > link); + } + + @Test + public void gettingStartedWebCloud() throws Exception { + runWithoutKey(GettingStartedWebCloud.class); + } +} diff --git a/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/HighEntropyValuesTest.java b/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/HighEntropyValuesTest.java new file mode 100644 index 000000000..2f3308288 --- /dev/null +++ b/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/HighEntropyValuesTest.java @@ -0,0 +1,74 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.web; + +import fiftyone.devicedetection.examples.web.GettingStartedWebCloud; +import org.junit.Test; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; + +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * The cloud web example illustrates the ways it obtains high entropy values + * and that they go on to the cloud service + * (device-detection-specification/required-examples.md, "The example SHOULD + * illustrate various techniques for obtaining "high entropy values" for + * presentation to the origin server as well as to the 51degrees cloud + * service"): the Accept-CH response header and the client-side JavaScript, + * which reads them with getHighEntropyValues and sends them back to the + * server, which presents them to the cloud service. + */ +public class HighEntropyValuesTest { + + private static String read(String file) throws Exception { + File found = getFilePath(GettingStartedWebCloud.resourceBase + file); + return new String(Files.readAllBytes(found.toPath()), + StandardCharsets.UTF_8); + } + + @Test + public void pageNamesEachTechnique() throws Exception { + String page = read("/WEB-INF/html/example-description.html"); + assertTrue("The page should name the Accept-CH header", + page.contains("Accept-CH")); + assertTrue("The page should name getHighEntropyValues, which the " + + "client-side JavaScript calls", page.contains("getHighEntropyValues")); + assertTrue("The page should say the values go on to the cloud service", + page.contains("cloud service")); + } + + @Test + public void optionsFileAddsTheJavaScriptAndItsEndpoint() throws Exception { + String options = read("/WEB-INF/51Degrees-Cloud.xml"); + assertTrue(options.contains("JavaScriptBuilderElement")); + assertTrue(options.contains("JsonBuilderElement")); + assertFalse("getHighEntropyValues needs a secure origin; the comment " + + "should not say it fails in one", + options.replaceAll("\\s+", " ").contains("in secure origins won't")); + } +} diff --git a/examples/device-detection/web/getting-started.onprem/pom.xml b/examples/device-detection/web/getting-started.onprem/pom.xml index 647fceabb..7f2b23d45 100644 --- a/examples/device-detection/web/getting-started.onprem/pom.xml +++ b/examples/device-detection/web/getting-started.onprem/pom.xml @@ -66,6 +66,13 @@ device-detection-java-examples.shared ${project.version} + + ${project.groupId} + device-detection.shared + ${device-detection.version} + test-jar + test + ch.qos.logback logback-classic diff --git a/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java b/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java index e74dec2b1..75bc81b53 100644 --- a/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java +++ b/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java @@ -53,49 +53,56 @@ * the free Lite file in cxx/data/device-detection is used, which is downloaded * before the server starts if it is missing. *

    - * The server listens on port 8081 and stops when Enter is pressed. If the PORT + * The server listens on port 5151 and stops when Enter is pressed. If the PORT * environment variable is set, the server listens on that port instead and runs * until the process is stopped, which is how automated tests start it. */ public class GettingStartedWebOnPrem extends HttpServlet { private static final long serialVersionUID = 1734154705981153540L; + /** + * The port the example listens on when the PORT environment variable + * does not choose another. + */ + public static final int DEFAULT_PORT = 5151; public static String resourceBase = "examples/device-detection/web/getting-started.onprem/src/main/webapp"; public static Logger logger = LoggerFactory.getLogger(GettingStartedWebOnPrem.class); - private static final String DD_PATH_VARIABLE = "_51DEGREES_DD_PATH"; - private static final String OLD_DD_PATH_VARIABLE = "51DEGREES_DD_PATH"; + public static void main(String[] args) throws Exception { + configureLogback(getFilePath("logback.xml")); + logger.info("Running Example {}", GettingStartedWebOnPrem.class); - /** - * Logs a warning when the configuration takes the data file from the - * deprecated name, which it does when that name is set and the name - * that replaces it is not. - */ - private static void warnOfOldDataFileName() { - if (isSet(OLD_DD_PATH_VARIABLE) && isSet(DD_PATH_VARIABLE) == false) { - logger.warn( - "The environment variable " + OLD_DD_PATH_VARIABLE - + " is deprecated and " + DD_PATH_VARIABLE - + " replaces it. Set " + DD_PATH_VARIABLE - + " to the same value, as a later release may no longer " - + "read " + OLD_DD_PATH_VARIABLE + "."); + String portEnv = System.getenv("PORT"); + if (portEnv != null) { + // Automated runs (e.g. the unified Selenium suite) inject a port and have no + // interactive stdin, so keep the server alive by joining it instead of waiting on Enter. + start(Integer.parseInt(portEnv)).join(); + } else { + // start Jetty with this WebApp + findDataFile(); + EmbedJetty.runWebApp(getFilePath(resourceBase).getAbsolutePath(), DEFAULT_PORT); } } - private static boolean isSet(String name) { - String value = System.getenv(name); - if (value == null || value.isEmpty()) { - value = System.getProperty(name); - } - return value != null && value.isEmpty() == false; + /** + * Finds the data file and starts the server with this WebApp, as main + * does, and returns the server, which runs until it is stopped. A test + * starts the example this way, with nothing set, to run it as it runs + * from a fresh clone. + * @param port the port to listen on + * @return the server + * @throws Exception if the data file cannot be found or the server + * cannot start + */ + public static org.eclipse.jetty.server.Server start(int port) throws Exception { + findDataFile(); + return EmbedJetty.startWebApp(getFilePath(resourceBase).getAbsolutePath(), port); } - public static void main(String[] args) throws Exception { - configureLogback(getFilePath("logback.xml")); - logger.info("Running Example {}", GettingStartedWebOnPrem.class); - warnOfOldDataFileName(); - - // If the data file the configuration names is not at that path, it - // is searched for, and the free Lite file is downloaded if missing. + /** + * If the data file the configuration names is not at that path, it is + * searched for, and the free Lite file is downloaded if missing. + */ + private static void findDataFile() throws Exception { String dataFile = PipelineOptionsFactory.getOptionsFromFile( getFilePath(resourceBase) + "/WEB-INF/51Degrees-OnPrem.xml") .findAndSubstitute("DeviceDetectionHashEngine", "DataFile"); @@ -103,16 +110,6 @@ public static void main(String[] args) throws Exception { System.setProperty("TestDataFile", DataFileHelper.getDataFileLocation(dataFile)); } - - String portEnv = System.getenv("PORT"); - if (portEnv != null) { - // Automated runs (e.g. the unified Selenium suite) inject a port and have no - // interactive stdin, so keep the server alive by joining it instead of waiting on Enter. - EmbedJetty.startWebApp(getFilePath(resourceBase).getAbsolutePath(), Integer.parseInt(portEnv)).join(); - } else { - // start Jetty with this WebApp - EmbedJetty.runWebApp(getFilePath(resourceBase).getAbsolutePath(), 8081); - } } FlowDataProviderCore flowDataProvider = new FlowDataProviderCore.Default(); @@ -127,6 +124,9 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re throws Exception { // the detection has already been carried out by the Filter // which is responsible to the lifecycle of the flowData - do NOT dispose + // A Flow Data cannot be used by two threads at once. Work handed to + // another thread or task gets the values it needs, read here on the + // request's thread, not the Flow Data. FlowData flowData = flowDataProvider.getFlowData(request); // retrieve the device data from the flowdata DeviceData device = flowData.get(DeviceData.class); diff --git a/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml b/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml index f6bc4c25d..1a4d4854c 100644 --- a/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml +++ b/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml @@ -29,11 +29,10 @@ false - ${_51DEGREES_DD_PATH:-${51DEGREES_DD_PATH:-${TestDataFile:-cxx/data/device-detection/51Degrees-LiteV4.1.hash}}} + ${_51DEGREES_DD_PATH:-${TestDataFile:-cxx/data/device-detection/51Degrees-LiteV4.1.hash}} LowMemory DeviceDetectionHashEngine @@ -49,8 +48,9 @@ JavaScriptBuilderElement http diff --git a/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/html/example-description.html b/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/html/example-description.html index f078f19f8..ccc5cf91f 100644 --- a/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/html/example-description.html +++ b/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/html/example-description.html @@ -23,10 +23,17 @@

    This example demonstrates the use of the Pipeline API to perform device detection in a simple Web Application servlet project.

    In particular, it highlights -

    1. Automatic handling of the 'Accept-CH' header, which is used - to request User-Agent Client Hints from the browser;
    2. -
    3. Client-side evidence collection in order to identify Apple device - models and properties such as screen size.
    +
    1. Automatic handling of the 'Accept-CH' header, which is used + to request User-Agent Client Hints from the browser. Chromium browsers + then send the high entropy values, such as the platform version and the + device model, as headers with their next request to this server;
    2. +
    3. Client-side evidence collection. The 51Degrees JavaScript reads the + high entropy values with navigator.userAgentData.getHighEntropyValues + and sends them back to this server, which detects again with them, + so the first page a browser asks for gets them too. The same script + identifies Apple device models and properties such as screen size. + Browsers give high entropy values only to a secure (https) origin + or localhost.

    The examples GettingStartedWebOnPrem and GettingStartedWebCloud differ only in their use of on-premise and cloud detection. diff --git a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/DefaultPortTest.java b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/DefaultPortTest.java new file mode 100644 index 000000000..0695b981a --- /dev/null +++ b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/DefaultPortTest.java @@ -0,0 +1,38 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.web; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Checks that the example listens on port 5151 when the PORT environment + * variable does not choose another, as the readme says. + */ +public class DefaultPortTest { + @Test + public void listensOn5151ByDefault() { + assertEquals(5151, GettingStartedWebOnPrem.DEFAULT_PORT); + } +} diff --git a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java index d6a94896f..569cf96e3 100644 --- a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java +++ b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPremTest.java @@ -22,7 +22,8 @@ package fiftyone.devicedetection.examples.web; -import fiftyone.devicedetection.shared.testhelpers.FileUtils; +import fiftyone.devicedetection.examples.shared.DataFileHelper; +import fiftyone.devicedetection.examples.shared.FindOutMore; import org.eclipse.jetty.server.Server; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -36,32 +37,41 @@ import static fiftyone.devicedetection.examples.web.GettingStartedWebOnPrem.resourceBase; import static fiftyone.pipeline.util.FileFinder.getFilePath; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class GettingStartedWebOnPremTest { private static Server SERVER; @BeforeClass public static void startJetty() throws Exception { - System.setProperty("TestDataFile", FileUtils.getHashFile() == null - ? FileUtils.LITE_HASH_DATA_FILE_PATH - : FileUtils.getHashFile().getAbsolutePath()); - SERVER = EmbedJetty.startWebApp(getFilePath(resourceBase).getAbsolutePath(), 8081); + // With no Enterprise file, the free Lite file is found, or + // downloaded, as the example finds it, so a fresh clone runs this. + System.setProperty( + "TestDataFile", DataFileHelper.getDataFileLocation(null)); + // Listen on the example's default port, so the request below to + // port 5151 checks that the example listens there. + SERVER = EmbedJetty.startWebApp(getFilePath(resourceBase).getAbsolutePath(), + GettingStartedWebOnPrem.DEFAULT_PORT); } @Test public void testWebOnPrem() throws Exception { HttpURLConnection connection = - (HttpURLConnection) new URL("http://localhost:8081/").openConnection(); + (HttpURLConnection) new URL("http://localhost:5151/").openConnection(); InputStream response = connection.getInputStream(); + String responseBody; try (Scanner scanner = new Scanner(response)) { - String responseBody = scanner.useDelimiter("\\A").next(); + responseBody = scanner.useDelimiter("\\A").next(); System.out.println(responseBody); } int code = connection.getResponseCode(); connection.disconnect(); assertEquals(200, code); + assertTrue("The page should end with the '" + FindOutMore.HEADING + + "' section", responseBody.contains( + "

    " + FindOutMore.HEADING + "

    ")); } @AfterClass diff --git a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/HighEntropyValuesTest.java b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/HighEntropyValuesTest.java new file mode 100644 index 000000000..5250e3f6d --- /dev/null +++ b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/HighEntropyValuesTest.java @@ -0,0 +1,71 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.web; + +import org.junit.Test; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; + +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * The web example illustrates the ways it obtains high entropy values + * (device-detection-specification/required-examples.md, "The example SHOULD + * illustrate various techniques for obtaining "high entropy values" for + * presentation to the origin server as well as to the 51degrees cloud + * service"): the Accept-CH response header, which asks the browser to send + * them with its next request, and the client-side JavaScript, which reads + * them with getHighEntropyValues and sends them back to the server. The + * page says so, and the options file adds the elements the JavaScript needs. + */ +public class HighEntropyValuesTest { + + private static String read(String file) throws Exception { + File found = getFilePath(GettingStartedWebOnPrem.resourceBase + file); + return new String(Files.readAllBytes(found.toPath()), + StandardCharsets.UTF_8); + } + + @Test + public void pageNamesEachTechnique() throws Exception { + String page = read("/WEB-INF/html/example-description.html"); + assertTrue("The page should name the Accept-CH header", + page.contains("Accept-CH")); + assertTrue("The page should name getHighEntropyValues, which the " + + "client-side JavaScript calls", page.contains("getHighEntropyValues")); + } + + @Test + public void optionsFileAddsTheJavaScriptAndItsEndpoint() throws Exception { + String options = read("/WEB-INF/51Degrees-OnPrem.xml"); + assertTrue(options.contains("JavaScriptBuilderElement")); + assertTrue(options.contains("JsonBuilderElement")); + assertFalse("getHighEntropyValues needs a secure origin; the comment " + + "should not say it fails in one", + options.replaceAll("\\s+", " ").contains("in secure origins won't")); + } +} diff --git a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/NonDefaultSettingsTest.java b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/NonDefaultSettingsTest.java new file mode 100644 index 000000000..2b8cfd61b --- /dev/null +++ b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/NonDefaultSettingsTest.java @@ -0,0 +1,85 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.web; + +import org.junit.Test; +import org.w3c.dom.Comment; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.File; + +import static fiftyone.devicedetection.examples.web.GettingStartedWebOnPrem.resourceBase; +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * The on-premise web example sets and documents a commonly used setting + * that is not the default, suppressing exceptions from processing so that + * a failure does not stop a production web site + * (device-detection-specification/required-examples.md, "Examples MUST also + * demonstrate or at least document commonly used, but non-default + * settings"). + */ +public class NonDefaultSettingsTest { + + @Test + public void suppressProcessExceptionIsSetAndDocumented() throws Exception { + File file = getFilePath(resourceBase + "/WEB-INF/51Degrees-OnPrem.xml"); + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setFeature( + "http://apache.org/xml/features/disallow-doctype-decl", true); + Document document = factory.newDocumentBuilder().parse(file); + Element root = document.getDocumentElement(); + + // The pipeline's own build parameters, not an element's. + Element parameters = null; + NodeList children = root.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + Node child = children.item(i); + if (child instanceof Element && + "BuildParameters".equals(child.getNodeName())) { + parameters = (Element) child; + } + } + assertNotNull("The pipeline should have build parameters", parameters); + NodeList suppress = parameters.getElementsByTagName( + "SuppressProcessException"); + assertEquals(1, suppress.getLength()); + assertEquals("true", suppress.item(0).getTextContent().trim()); + + // A comment before the setting says what it does. + Node previous = parameters.getPreviousSibling(); + while (previous != null && previous instanceof Comment == false) { + previous = previous.getPreviousSibling(); + } + assertNotNull("The setting should be documented", previous); + assertTrue(previous.getTextContent(), + previous.getTextContent().contains("Suppress")); + } +} diff --git a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java index 7f9bc4df7..ebad722aa 100644 --- a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java +++ b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java @@ -41,35 +41,29 @@ */ public class OnPremDataFileOptionTest { private static final String DD_PATH = "_51DEGREES_DD_PATH"; - private static final String OLD_DD_PATH = "51DEGREES_DD_PATH"; private static final String TEST_DATA_FILE = "TestDataFile"; private static final String LITE = "cxx/data/device-detection/51Degrees-LiteV4.1.hash"; private String savedDdPath; - private String savedOldDdPath; private String savedTestDataFile; @Before public void saveProperties() { - // An environment variable of either name would take precedence over + // An environment variable of any of these names would take precedence over // the system properties set here, so the test cannot run. assumeTrue(System.getenv(DD_PATH) == null); - assumeTrue(System.getenv(OLD_DD_PATH) == null); assumeTrue(System.getenv(TEST_DATA_FILE) == null); assumeTrue(System.getenv(TEST_DATA_FILE.toUpperCase()) == null); savedDdPath = System.getProperty(DD_PATH); - savedOldDdPath = System.getProperty(OLD_DD_PATH); savedTestDataFile = System.getProperty(TEST_DATA_FILE); System.clearProperty(DD_PATH); - System.clearProperty(OLD_DD_PATH); System.clearProperty(TEST_DATA_FILE); } @After public void restoreProperties() { restore(DD_PATH, savedDdPath); - restore(OLD_DD_PATH, savedOldDdPath); restore(TEST_DATA_FILE, savedTestDataFile); } @@ -91,20 +85,6 @@ public void prefersDdPathOverTestDataFile() throws Exception { assertEquals("from-dd-path.hash", dataFile()); } - @Test - public void usesOldDdPathWhenNewIsNotSet() throws Exception { - System.setProperty(TEST_DATA_FILE, "from-test-data-file.hash"); - System.setProperty(OLD_DD_PATH, "from-old-dd-path.hash"); - assertEquals("from-old-dd-path.hash", dataFile()); - } - - @Test - public void prefersDdPathOverOldDdPath() throws Exception { - System.setProperty(OLD_DD_PATH, "from-old-dd-path.hash"); - System.setProperty(DD_PATH, "from-dd-path.hash"); - assertEquals("from-dd-path.hash", dataFile()); - } - private static String dataFile() throws Exception { PipelineOptions options = PipelineOptionsFactory.getOptionsFromFile( getFilePath(resourceBase) + "/WEB-INF/51Degrees-OnPrem.xml"); diff --git a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/WebOnPremWithoutSettingsTest.java b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/WebOnPremWithoutSettingsTest.java new file mode 100644 index 000000000..aa91c409b --- /dev/null +++ b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/WebOnPremWithoutSettingsTest.java @@ -0,0 +1,85 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.web; + +import fiftyone.devicedetection.examples.shared.FindOutMore; +import org.eclipse.jetty.server.Server; +import org.junit.Test; + +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Scanner; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +/** + * Starts the on-premise web example as its main method does, with no + * arguments and no data file set, as someone who has just cloned the + * repository runs it, so that it uses the free Lite data file. The other + * test of this example gives it the Enterprise data file when CI has one, + * which does not show that the example works out of the box + * (device-detection-specification/required-examples.md, "Those tests MUST + * include a run of each example with no arguments or settings"). + */ +public class WebOnPremWithoutSettingsTest { + + private static final int PORT = 8083; + + private static final String[] DATA_FILE_NAMES = { + "TestDataFile", "_51DEGREES_DD_PATH"}; + + @Test + public void runsWithoutSettings() throws Exception { + for (String name : DATA_FILE_NAMES) { + assumeTrue("Skipping, " + name + " is set in the environment", + System.getenv(name) == null); + } + String saved = System.clearProperty("TestDataFile"); + Server server = GettingStartedWebOnPrem.start(PORT); + try { + HttpURLConnection connection = (HttpURLConnection) + new URL("http://localhost:" + PORT + "/").openConnection(); + String body; + try (InputStream response = connection.getInputStream(); + Scanner scanner = new Scanner(response, "UTF-8")) { + body = scanner.useDelimiter("\\A").next(); + } + int code = connection.getResponseCode(); + connection.disconnect(); + assertEquals(200, code); + assertTrue("The page should say it used the free Lite data " + + "file. " + body, body.contains("Lite")); + assertTrue("The page should end with the '" + + FindOutMore.HEADING + "' section. " + body, + body.contains("

    " + FindOutMore.HEADING + "

    ")); + } finally { + server.stop(); + if (saved != null) { + System.setProperty("TestDataFile", saved); + } + } + } +} diff --git a/examples/device-detection/web/shared/src/main/java/fiftyone/devicedetection/examples/web/HtmlContentHelper.java b/examples/device-detection/web/shared/src/main/java/fiftyone/devicedetection/examples/web/HtmlContentHelper.java index 2df9b35a9..d6184ce2a 100644 --- a/examples/device-detection/web/shared/src/main/java/fiftyone/devicedetection/examples/web/HtmlContentHelper.java +++ b/examples/device-detection/web/shared/src/main/java/fiftyone/devicedetection/examples/web/HtmlContentHelper.java @@ -23,6 +23,7 @@ package fiftyone.devicedetection.examples.web; import fiftyone.devicedetection.examples.shared.DataFileHelper; +import fiftyone.devicedetection.examples.shared.FindOutMore; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.shared.DeviceData; import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; @@ -49,6 +50,12 @@ import static fiftyone.pipeline.util.FileFinder.getFilePath; public class HtmlContentHelper { + /** + * The tag of the links in the pages' "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "web-shared-src-main-java-fiftyone-devicedetection-examples-web-htmlcontenthelper.java"; + /** * Helper to output HTML headers * @param out the PrintWriter to write to @@ -306,6 +313,10 @@ private static void doContactUsMessage(PrintWriter out, boolean cloud) { public static void doHtmlPostamble(PrintWriter out, FlowData flowData) { // language=html doContactUsMessage(out, flowData); + out.append(FindOutMore.html(EXAMPLE_TAG, + getFlowElement(flowData) instanceof CloudRequestEngine ? + FindOutMore.CLOUD_REPOSITORIES : + FindOutMore.ON_PREMISE_REPOSITORIES)); out.append("\n\n\n"); } diff --git a/examples/ip-intelligence/README.md b/examples/ip-intelligence/README.md index e3775778c..0d7dbf5b9 100644 --- a/examples/ip-intelligence/README.md +++ b/examples/ip-intelligence/README.md @@ -90,10 +90,10 @@ The tables below describe the examples available in this repository. ### Cloud -Cloud examples require a resource key, supplied via the `TestResourceKey` -environment variable, system property, or command line argument. Resource keys +Cloud examples require a resource key, supplied via the `_51DEGREES_RESOURCE_KEY` +environment variable or system property, or a command line argument. Resource keys for IP Intelligence are not yet generally available, so these examples can also -be pointed at a custom hosted cloud endpoint using the `TestCloudEndPoint` +be pointed at a custom hosted cloud endpoint using the `_51DEGREES_CLOUD_ENDPOINT` environment variable or system property. | Example | Description | @@ -104,8 +104,8 @@ environment variable or system property. | MetadataCloud | How to access the meta-data that relates to the properties available from the cloud IP intelligence service. | Cloud examples need a resource key. The key is read from the -`_51DEGREES_RESOURCE_KEY` environment variable or system property first. The -legacy `TestResourceKey` name is still supported and is checked second. +`_51DEGREES_RESOURCE_KEY` environment variable or system property, and no +other name. The cloud property tiers changed in May 2026. The examples and this documentation now reflect what is free and what needs a paid subscription. @@ -117,9 +117,9 @@ documentation now reflect what is free and what needs a paid subscription. and AccuracyRadiusMin. A free resource key selecting the free tier properties can be created at -https://configure.51degrees.com/Wkqxf3Bs?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon. A resource key that also includes +https://51degrees.com/configure/Wkqxf3Bs?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon. A resource key that also includes the paid properties used by the examples can be created at -https://configure.51degrees.com/hYzn3TV3?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon. See https://51degrees.com/pricing?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon +https://51degrees.com/configure/hYzn3TV3?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon. See https://51degrees.com/pricing?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon to get a paid subscription with more properties. ### On-Premise @@ -185,7 +185,7 @@ Web examples: mvn compile exec:java -pl web/getting-started.onprem -Dexec.mainClass="fiftyone.ipintelligence.examples.web.GettingStartedWebOnPrem" ``` -The web example will start a Jetty server on port 8081. Access it at: `http://localhost:8081` +The web example will start a Jetty server on port 5151. Access it at: `http://localhost:5151` ### Running Examples from Packaged JARs @@ -199,8 +199,22 @@ mvn package This will produce JARs with dependencies inside `target` subfolders. -Then run examples using the JAR: +Then run examples using the JAR, from this folder. The version in the name is +the one in this folder's `pom.xml`, which is not the version of the packages +the examples use. ```bash -java -cp .\console\target\ip-intelligence-java-examples.console-4.4.19-jar-with-dependencies.jar fiftyone.ipintelligence.examples.console.OfflineProcessing +java -cp ./console/target/ip-intelligence-java-examples.console-4.5.127-SNAPSHOT-jar-with-dependencies.jar fiftyone.ipintelligence.examples.console.OfflineProcessing ``` + +## Flow Data and threads + +A Flow Data cannot be used by two threads at once. The web examples handle +each request on one thread, and read the Flow Data there. An application that +hands work from a request to another thread or task reads the values that +work needs from the Flow Data on the request's thread, into its own +variables, and passes those values, not the Flow Data. The Flow Data is +closed when the request ends, so work that outlives the request must not +keep it either. The +[pipeline README](../../pipeline/README.md#flow-data-and-threads) +shows such a hand-off. diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java index 0fc3c7bbf..429992ff1 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java @@ -72,6 +72,7 @@ package fiftyone.ipintelligence.examples.console; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.ipintelligence.IPIntelligenceOnPremisePipelineBuilder; import fiftyone.ipintelligence.IPIntelligencePipelineBuilder; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; @@ -96,6 +97,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -115,6 +117,12 @@ * location is contained in the area returned are written to an output CSV. */ public class CompareOnPrem { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-ipintelligence-examples-console-compareonprem.java"; + private static final Logger logger = LoggerFactory.getLogger(CompareOnPrem.class); /** @@ -207,6 +215,8 @@ public static void main(String[] args) throws Exception { args[3], new PrintWriter(System.out)) : DEFAULT_PERFORMANCE_PROFILE; if (profile == null) { + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); return; } @@ -216,6 +226,8 @@ args[3], new PrintWriter(System.out)) : LOCATION_PROPERTIES, new PrintWriter(System.out), CONTACT_US)) { + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); return; } @@ -223,6 +235,8 @@ args[3], new PrintWriter(System.out)) : run(dataFile, csvTruthFile, profile, writer); } logger.info("Output written to '{}'", outputFile); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); } /** @@ -273,7 +287,9 @@ public static void run(String dataFile, // for more detail on this and other configuration options. // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-compareonprem.java&utm_term=automatic-datafile-updates .setPerformanceProfile(profile) - // inhibit sharing usage for this example + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. .setShareUsage(false) // inhibit auto-update of the data file for this example .setAutoUpdate(false) @@ -421,13 +437,22 @@ private static String processTruth(Pipeline pipeline, Truth truth) latitude + "," + longitude + "," + (confidence != null ? confidence : "") + "," + - String.format("%.3f", distanceKms) + "," + + kilometres(distanceKms) + "," + area.getSquareKms() + "," + area.getGeometries() + "," + area.getContains(); } } + /** + * A distance to three decimal places with a full stop as the separator + * whatever the default locale, as a decimal comma would split the CSV + * column in two. + */ + static String kilometres(double distanceKms) { + return String.format(Locale.ROOT, "%.3f", distanceKms); + } + /** * Returns the great circle distance in kilometers between two points * using the haversine formula. diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloud.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloud.java index 3a55152aa..e8d7d7cf0 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloud.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloud.java @@ -46,6 +46,7 @@ package fiftyone.ipintelligence.examples.console; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.ipintelligence.IPIntelligenceCloudPipelineBuilder; import fiftyone.ipintelligence.IPIntelligencePipelineBuilder; import fiftyone.ipintelligence.examples.shared.KeyHelper; @@ -71,11 +72,17 @@ *

    * Access to the Cloud service is configured using a "resource key". The resource key can be * supplied as a command line argument, or via the environment variable or system property - * "TestResourceKey". An optional second command line argument, or the environment variable or - * system property "TestCloudEndPoint", can be used to direct the example at a custom hosted + * "_51DEGREES_RESOURCE_KEY". An optional second command line argument, or the environment variable or + * system property "_51DEGREES_CLOUD_ENDPOINT", can be used to direct the example at a custom hosted * Cloud service. */ public class GetAllPropertiesCloud { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-ipintelligence-examples-console-getallpropertiescloud.java"; + private static final Logger logger = LoggerFactory.getLogger(GetAllPropertiesCloud.class); /** @@ -88,13 +95,21 @@ public static void main(String[] args) throws Exception { // Use the resource key from the command line, environment variable or // system property. - String resourceKey = args.length > 0 ? args[0] : - KeyHelper.getOrSetTestResourceKey(); + String resourceKey = KeyHelper.findResourceKey( + args.length > 0 ? args[0] : null); + if (resourceKey == null) { + KeyHelper.writeNoResourceKey(System.out, EXAMPLE_TAG); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); + return; + } // An optional custom endpoint for a self-hosted Cloud service. String endPoint = args.length > 1 ? args[1] : KeyUtils.getNamedKey(GettingStartedCloud.CLOUD_END_POINT_NAME); run(resourceKey, endPoint, System.out); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); } /** diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java index 684990a91..427261eaa 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloud.java @@ -46,6 +46,7 @@ package fiftyone.ipintelligence.examples.console; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.ipintelligence.IPIntelligenceCloudPipelineBuilder; import fiftyone.ipintelligence.IPIntelligencePipelineBuilder; import fiftyone.ipintelligence.examples.shared.EvidenceHelper; @@ -53,7 +54,9 @@ import fiftyone.ipintelligence.shared.IPIntelligenceData; import fiftyone.ipintelligence.shared.testhelpers.KeyUtils; import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.IWeightedValue; import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.data.AspectPropertyValue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,12 +66,9 @@ import java.util.Map; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asFloatProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asIPAddressProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asIntegerProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asStringProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asWktStringProperty; +import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asWeightedString; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.tryGet; +import static fiftyone.ipintelligence.examples.shared.PropertyHelper.weightedValues; import static fiftyone.pipeline.util.FileFinder.getFilePath; /** @@ -78,28 +78,40 @@ *

    * Access to the Cloud service is configured using a "resource key". The resource key can be * supplied as a command line argument, or via the environment variable or system property - * "TestResourceKey". An optional second command line argument, or the environment variable or - * system property "TestCloudEndPoint", can be used to direct the example at a custom hosted + * "_51DEGREES_RESOURCE_KEY". An optional second command line argument, or the environment variable or + * system property "_51DEGREES_CLOUD_ENDPOINT", can be used to direct the example at a custom hosted * Cloud service. *

    * The concepts of "pipeline", "flow data", "evidence" and "results" are illustrated. */ public class GettingStartedCloud { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedcloud.java"; + private static final Logger logger = LoggerFactory.getLogger(GettingStartedCloud.class); /** * Name of the environment variable or system property that can hold a * custom Cloud service endpoint URL. */ - public static final String CLOUD_END_POINT_NAME = "TestCloudEndPoint"; + public static final String CLOUD_END_POINT_NAME = "_51DEGREES_CLOUD_ENDPOINT"; public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); // Use the resource key from the command line, environment variable or // system property. - String resourceKey = args.length > 0 ? args[0] : - KeyHelper.getOrSetTestResourceKey(); + String resourceKey = KeyHelper.findResourceKey( + args.length > 0 ? args[0] : null); + if (resourceKey == null) { + KeyHelper.writeNoResourceKey(System.out, EXAMPLE_TAG); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); + return; + } // An optional custom endpoint for a self-hosted Cloud service. String endPoint = args.length > 1 ? args[1] : KeyUtils.getNamedKey(CLOUD_END_POINT_NAME); @@ -107,6 +119,8 @@ public static void main(String[] args) throws Exception { // prepare 'evidence' for use in pipeline (see below) List> evidence = EvidenceHelper.setUpEvidence(); run(resourceKey, endPoint, evidence, System.out); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); } /** @@ -190,24 +204,51 @@ private static void analyzeEvidence(Map evidence, /* Display the results of the lookup, which are called IP Intelligence properties. See the property dictionary at https://51degrees.com/developers/property-dictionary?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedcloud.java&utm_term=property-dictionary for details of all available properties. */ - writer.println("\tRegisteredName: " + asStringProperty(tryGet(ipData::getRegisteredName))); - writer.println("\tRegisteredOwner: " + asStringProperty(tryGet(ipData::getRegisteredOwner))); - writer.println("\tRegisteredCountry: " + asStringProperty(tryGet(ipData::getRegisteredCountry))); - writer.println("\tIpRangeStart: " + asIPAddressProperty(tryGet(ipData::getIpRangeStart))); - writer.println("\tIpRangeEnd: " + asIPAddressProperty(tryGet(ipData::getIpRangeEnd))); - writer.println("\tCountry: " + asStringProperty(tryGet(ipData::getCountry))); - writer.println("\tCountryCode: " + asStringProperty(tryGet(ipData::getCountryCode))); - writer.println("\tCountryCode3: " + asStringProperty(tryGet(ipData::getCountryCode3))); - writer.println("\tRegion: " + asStringProperty(tryGet(ipData::getRegion))); - writer.println("\tState: " + asStringProperty(tryGet(ipData::getState))); - writer.println("\tTown: " + asStringProperty(tryGet(ipData::getTown))); - writer.println("\tLatitude: " + asFloatProperty(tryGet(ipData::getLatitude))); - writer.println("\tLongitude: " + asFloatProperty(tryGet(ipData::getLongitude))); - writer.println("\tAreas: " + asWktStringProperty(tryGet(ipData::getAreas))); - writer.println("\tAccuracyRadiusMin: " + asIntegerProperty(tryGet(ipData::getAccuracyRadiusMin))); - writer.println("\tTimeZoneOffset: " + asIntegerProperty(tryGet(ipData::getTimeZoneOffset))); + writeValue(writer, "RegisteredName", tryGet(ipData::getRegisteredName)); + writeValue(writer, "RegisteredOwner", tryGet(ipData::getRegisteredOwner)); + writeValue(writer, "RegisteredCountry", tryGet(ipData::getRegisteredCountry)); + writeValue(writer, "IpRangeStart", tryGet(ipData::getIpRangeStart)); + writeValue(writer, "IpRangeEnd", tryGet(ipData::getIpRangeEnd)); + writeValue(writer, "Country", tryGet(ipData::getCountry)); + writeValue(writer, "CountryCode", tryGet(ipData::getCountryCode)); + writeValue(writer, "CountryCode3", tryGet(ipData::getCountryCode3)); + writeValue(writer, "Region", tryGet(ipData::getRegion)); + writeValue(writer, "State", tryGet(ipData::getState)); + writeValue(writer, "Town", tryGet(ipData::getTown)); + writeValue(writer, "Latitude", tryGet(ipData::getLatitude)); + writeValue(writer, "Longitude", tryGet(ipData::getLongitude)); + writeValue(writer, "Areas", tryGet(ipData::getAreas)); + writeValue(writer, "AccuracyRadiusMin", tryGet(ipData::getAccuracyRadiusMin)); + writeValue(writer, "TimeZoneOffset", tryGet(ipData::getTimeZoneOffset)); } writer.println(); writer.flush(); } + + /** + * Writes each value of a property with its weighting, or why it has no + * value. + */ + private static void writeValue(PrintWriter writer, + String property, + AspectPropertyValue result) { + if (result.hasValue() == false) { + writer.println("\t" + property + ": " + asWeightedString(result)); + return; + } + /* IP Intelligence properties return weighted values. Each value comes + with a weighting between 0 and 1, which says how much of the IP range + the value applies to, and a property with a single answer returns it + with a weighting of 1. */ + StringBuilder line = new StringBuilder(); + for (IWeightedValue weighted : weightedValues(result)) { + if (line.length() > 0) { + line.append(", "); + } + line.append(asWeightedString( + weighted.getValue(), + weighted.getWeighting())); + } + writer.println("\t" + property + ": " + line); + } } diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java index efca03fdc..571299223 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixed.java @@ -47,6 +47,7 @@ package fiftyone.ipintelligence.examples.console; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.devicedetection.cloud.flowelements.DeviceDetectionCloudEngine; import fiftyone.devicedetection.cloud.flowelements.DeviceDetectionCloudEngineBuilder; import fiftyone.devicedetection.shared.DeviceData; @@ -85,11 +86,17 @@ *

    * Access to the Cloud service is configured using a "resource key". The resource key can be * supplied as a command line argument, or via the environment variable or system property - * "TestResourceKey". An optional second command line argument, or the environment variable or - * system property "TestCloudEndPoint", can be used to direct the example at a custom hosted + * "_51DEGREES_RESOURCE_KEY". An optional second command line argument, or the environment variable or + * system property "_51DEGREES_CLOUD_ENDPOINT", can be used to direct the example at a custom hosted * Cloud service. */ public class GettingStartedCloudMixed { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedcloudmixed.java"; + private static final Logger logger = LoggerFactory.getLogger(GettingStartedCloudMixed.class); public static void main(String[] args) throws Exception { @@ -97,8 +104,14 @@ public static void main(String[] args) throws Exception { // Use the resource key from the command line, environment variable or // system property. - String resourceKey = args.length > 0 ? args[0] : - KeyHelper.getOrSetTestResourceKey(); + String resourceKey = KeyHelper.findResourceKey( + args.length > 0 ? args[0] : null); + if (resourceKey == null) { + KeyHelper.writeNoResourceKey(System.out, EXAMPLE_TAG); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); + return; + } // An optional custom endpoint for a self-hosted Cloud service. String endPoint = args.length > 1 ? args[1] : KeyUtils.getNamedKey(GettingStartedCloud.CLOUD_END_POINT_NAME); @@ -107,6 +120,8 @@ public static void main(String[] args) throws Exception { // includes both a User-Agent header and an IP address. List> evidence = GettingStartedMixed.setUpCombinedEvidence(); run(resourceKey, endPoint, evidence, System.out); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); } /** diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java index f007e2fb3..69cba6a13 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java @@ -47,6 +47,7 @@ package fiftyone.ipintelligence.examples.console; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.shared.DeviceData; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; @@ -85,6 +86,12 @@ * The configuration file is src/main/resources/gettingStartedMixed.xml. */ public class GettingStartedMixed { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedmixed.java"; + private static final Logger logger = LoggerFactory.getLogger(GettingStartedMixed.class); public static final String DD_DATA_FILE_DEFAULT = "device-detection/51Degrees-LiteV4.1.hash"; @@ -105,6 +112,8 @@ public static void main(String[] args) throws Exception { List> evidence = setUpCombinedEvidence(); run(ddDataFile, ipiDataFile, evidence, System.out); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); } /** @@ -147,6 +156,10 @@ public static void run(String ddDataFile, PipelineOptions pipelineOptions = PipelineOptionsFactory.getOptionsFromFile(optionsFile); try (Pipeline pipeline = new FiftyOnePipelineBuilder() + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. + .setShareUsage(false) .buildFromConfiguration(pipelineOptions)) { // The engines say which properties their data files have. diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java index e25f3e109..6cf2f321f 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java @@ -44,6 +44,7 @@ package fiftyone.ipintelligence.examples.console; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.ipintelligence.engine.onpremise.interop.swig.WeightedStringListValueSwig; import fiftyone.ipintelligence.examples.shared.DataFileHelper; import fiftyone.ipintelligence.examples.shared.EvidenceHelper; @@ -64,10 +65,11 @@ import java.io.PrintWriter; import java.util.List; import java.util.Map; -import java.util.function.Supplier; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asString; +import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asWeightedString; +import static fiftyone.ipintelligence.examples.shared.PropertyHelper.weightedValues; import static fiftyone.pipeline.util.FileFinder.getFilePath; import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; @@ -82,6 +84,12 @@ * The concepts of "pipeline", "flow data", "evidence" and "results" are illustrated. */ public class GettingStartedOnPrem { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java"; + private static final Logger logger = LoggerFactory.getLogger(GettingStartedOnPrem.class); /* In this example, by default, the free 51degrees Lite IP Intelligence data file is used from @@ -104,6 +112,8 @@ public static void main(String[] args) throws Exception { // prepare 'evidence' for use in pipeline (see below) List> evidence = EvidenceHelper.setUpEvidence(); run(dataFile, evidence, System.out); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); } /** @@ -147,6 +157,10 @@ public static void run(String dataFile, PipelineOptions pipelineOptions = PipelineOptionsFactory.getOptionsFromFile(optionsFile); // Build a new Pipeline from the configuration. try (Pipeline pipeline = new FiftyOnePipelineBuilder() + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. + .setShareUsage(false) .buildFromConfiguration(pipelineOptions)) { @@ -218,23 +232,24 @@ private static boolean analyzeEvidence(Map evidence, details of all available properties. */ // The free Lite data file has the registration properties, and - // only the Enterprise data file has the others. - needsEnterprise |= writeValue(writer, engine, "RegisteredName", ipData::getRegisteredName); - needsEnterprise |= writeValue(writer, engine, "RegisteredOwner", ipData::getRegisteredOwner); - needsEnterprise |= writeValue(writer, engine, "RegisteredCountry", ipData::getRegisteredCountry); - needsEnterprise |= writeValue(writer, engine, "IpRangeStart", ipData::getIpRangeStart); - needsEnterprise |= writeValue(writer, engine, "IpRangeEnd", ipData::getIpRangeEnd); - needsEnterprise |= writeValue(writer, engine, "Country", ipData::getCountry); - needsEnterprise |= writeValue(writer, engine, "CountryCode", ipData::getCountryCode); - needsEnterprise |= writeValue(writer, engine, "CountryCode3", ipData::getCountryCode3); - needsEnterprise |= writeValue(writer, engine, "Region", ipData::getRegion); - needsEnterprise |= writeValue(writer, engine, "State", ipData::getState); - needsEnterprise |= writeValue(writer, engine, "Town", ipData::getTown); - needsEnterprise |= writeValue(writer, engine, "Latitude", ipData::getLatitude); - needsEnterprise |= writeValue(writer, engine, "Longitude", ipData::getLongitude); - needsEnterprise |= writeValue(writer, engine, "Areas", ipData::getAreas); - needsEnterprise |= writeValue(writer, engine, "AccuracyRadiusMin", ipData::getAccuracyRadiusMin); - needsEnterprise |= writeValue(writer, engine, "TimeZoneOffset", ipData::getTimeZoneOffset); + // only the Enterprise data file has the others. Each value is + // read by name, which gives it with its weighting. + needsEnterprise |= writeValue(writer, engine, ipData, "RegisteredName"); + needsEnterprise |= writeValue(writer, engine, ipData, "RegisteredOwner"); + needsEnterprise |= writeValue(writer, engine, ipData, "RegisteredCountry"); + needsEnterprise |= writeValue(writer, engine, ipData, "IpRangeStart"); + needsEnterprise |= writeValue(writer, engine, ipData, "IpRangeEnd"); + needsEnterprise |= writeValue(writer, engine, ipData, "Country"); + needsEnterprise |= writeValue(writer, engine, ipData, "CountryCode"); + needsEnterprise |= writeValue(writer, engine, ipData, "CountryCode3"); + needsEnterprise |= writeValue(writer, engine, ipData, "Region"); + needsEnterprise |= writeValue(writer, engine, ipData, "State"); + needsEnterprise |= writeValue(writer, engine, ipData, "Town"); + needsEnterprise |= writeValue(writer, engine, ipData, "Latitude"); + needsEnterprise |= writeValue(writer, engine, ipData, "Longitude"); + needsEnterprise |= writeValue(writer, engine, ipData, "Areas"); + needsEnterprise |= writeValue(writer, engine, ipData, "AccuracyRadiusMin"); + needsEnterprise |= writeValue(writer, engine, ipData, "TimeZoneOffset"); } writer.println(); writer.flush(); @@ -245,14 +260,36 @@ private static boolean analyzeEvidence(Map evidence, * Writes a property, or that it needs the Enterprise data file when the * data file in use does not have it. The value is only read when the * data file has the property. + *

    + * IP Intelligence values are weighted. Reading a property by name gives + * a list of values, each with its weighting, and a property with one + * answer gives it with a weighting of 1. The typed getters, such as + * {@link IPIntelligenceData#getRegisteredName()}, give the value alone. * @return true if the property needs the Enterprise data file */ private static boolean writeValue(PrintWriter writer, IPIntelligenceOnPremiseEngine engine, - String property, - Supplier> value) { + IPIntelligenceData ipData, + String property) { if (DataFileHelper.hasProperty(engine, property)) { - writer.println("\t" + property + ": " + asString(value.get())); + AspectPropertyValue value = + (AspectPropertyValue) ipData.get(property); + if (value.hasValue() == false) { + writer.println("\t" + property + ": " + asString(value)); + return false; + } + // Each value comes with a weighting between 0 and 1, which says + // how much of the IP range the value applies to. + StringBuilder line = new StringBuilder(); + for (IWeightedValue weighted : weightedValues(value)) { + if (line.length() > 0) { + line.append(", "); + } + line.append(asWeightedString( + weighted.getValue(), + weighted.getWeighting())); + } + writer.println("\t" + property + ": " + line); return false; } writer.println("\t" + property + ": " + DataFileHelper.NEEDS_ENTERPRISE); diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataCloud.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataCloud.java index 00da31c62..83c49ef0c 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataCloud.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataCloud.java @@ -54,6 +54,7 @@ package fiftyone.ipintelligence.examples.console; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.ipintelligence.IPIntelligenceCloudPipelineBuilder; import fiftyone.ipintelligence.IPIntelligencePipelineBuilder; import fiftyone.ipintelligence.cloud.flowelements.IPIntelligenceCloudEngine; @@ -80,11 +81,17 @@ *

    * Access to the Cloud service is configured using a "resource key". The resource key can be * supplied as a command line argument, or via the environment variable or system property - * "TestResourceKey". An optional second command line argument, or the environment variable or - * system property "TestCloudEndPoint", can be used to direct the example at a custom hosted + * "_51DEGREES_RESOURCE_KEY". An optional second command line argument, or the environment variable or + * system property "_51DEGREES_CLOUD_ENDPOINT", can be used to direct the example at a custom hosted * Cloud service. */ public class MetadataCloud { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-ipintelligence-examples-console-metadatacloud.java"; + private static final Logger logger = LoggerFactory.getLogger(MetadataCloud.class); public static void main(String[] args) throws Exception { @@ -92,13 +99,21 @@ public static void main(String[] args) throws Exception { // Use the resource key from the command line, environment variable or // system property. - String resourceKey = args.length > 0 ? args[0] : - KeyHelper.getOrSetTestResourceKey(); + String resourceKey = KeyHelper.findResourceKey( + args.length > 0 ? args[0] : null); + if (resourceKey == null) { + KeyHelper.writeNoResourceKey(System.out, EXAMPLE_TAG); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); + return; + } // An optional custom endpoint for a self-hosted Cloud service. String endPoint = args.length > 1 ? args[1] : KeyUtils.getNamedKey(GettingStartedCloud.CLOUD_END_POINT_NAME); run(resourceKey, endPoint, System.out); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); } /** diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java index 223121f1d..2619ca2cf 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java @@ -21,6 +21,7 @@ * ********************************************************************* */ package fiftyone.ipintelligence.examples.console; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.ipintelligence.examples.shared.DataFileHelper; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; @@ -61,6 +62,12 @@ * [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metadataonprem.java&utm_term=header). */ public class MetadataOnPrem { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-ipintelligence-examples-console-metadataonprem.java"; + private static final Logger logger = LoggerFactory.getLogger(GettingStartedOnPrem.class); /* In this example, by default, the free 51degrees Lite IP Intelligence data file is used from @@ -79,6 +86,8 @@ public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); String dataFile = args.length > 0 ? args[0] : LITE_DATA_FILE_REL_PATH; run(dataFile, System.out); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); } diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java index 4730245ec..76566b923 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java @@ -54,6 +54,7 @@ package fiftyone.ipintelligence.examples.console; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.ipintelligence.IPIntelligencePipelineBuilder; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; import fiftyone.ipintelligence.examples.console.areas.Calculations; @@ -78,6 +79,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Random; @@ -105,6 +107,12 @@ * are output in CSV format. */ public class MetricsOnPrem { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-ipintelligence-examples-console-metricsonprem.java"; + private static final Logger logger = LoggerFactory.getLogger(MetricsOnPrem.class); /** @@ -325,6 +333,8 @@ public static void main(String[] args) throws Exception { args[3], new PrintWriter(System.out)) : DEFAULT_PERFORMANCE_PROFILE; if (profile == null) { + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); return; } @@ -334,6 +344,8 @@ args[3], new PrintWriter(System.out)) : Arrays.asList(PROPERTIES), new PrintWriter(System.out), CONTACT_US)) { + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); return; } @@ -345,6 +357,8 @@ args[3], new PrintWriter(System.out)) : run(dataFile, samplePercentage, 0, condition, profile, writer); } logger.info("Output written to '{}'", outputFile); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); } /** @@ -407,7 +421,9 @@ public static void run(String dataFile, // configuration options. // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metricsonprem.java&utm_term=automatic-datafile-updates .setPerformanceProfile(profile) - // inhibit sharing usage for this example + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. .setShareUsage(false) // inhibit auto-update of the data file for this example .setAutoUpdate(false) @@ -736,11 +752,20 @@ private static void writeCsv(PrintWriter output, Map groups) { metric.areaCount + "," + metric.getAverageAreaKm() + "," + metric.getEquivalentRadiusKm() + "," + - String.format("%.2f", metric.getAveragePolygons())); + twoDecimalPlaces(metric.getAveragePolygons())); } output.flush(); } + /** + * A number to two decimal places with a full stop as the separator + * whatever the default locale, as a decimal comma would split the CSV + * column in two. + */ + static String twoDecimalPlaces(double value) { + return String.format(Locale.ROOT, "%.2f", value); + } + /** * Quote a CSV field when it contains a comma or quote. */ diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java index 0357b64d8..79fe0f5dd 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java @@ -54,6 +54,7 @@ package fiftyone.ipintelligence.examples.console; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.ipintelligence.IPIntelligencePipelineBuilder; import fiftyone.ipintelligence.examples.shared.DataFileHelper; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; @@ -96,6 +97,12 @@ * using Performance Profile settings. */ public class OfflineProcessing { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-ipintelligence-examples-console-offlineprocessing.java"; + static final Logger logger = LoggerFactory.getLogger(OfflineProcessing.class); // By default the free 51degrees Lite IP Intelligence data file is used from // the data repository in cxx/data, and is downloaded if it is missing. @@ -170,8 +177,9 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws // that the pipeline is disposed when done try (Pipeline pipeline = new IPIntelligencePipelineBuilder() .useOnPremise(dataFileLocation, false) - // inhibit sharing usage for this test, usually - // this should be set "true" + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. .setShareUsage(false) // inhibit auto-update of the data file for this test .setAutoUpdate(false) @@ -269,6 +277,10 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws } // finish the last YAML document writer.write("...\n"); + // The section is written as comments, so the output stays a + // valid YAML stream. + FindOutMore.writeCommented(new PrintWriter(writer), "# ", + EXAMPLE_TAG, FindOutMore.ON_PREMISE_REPOSITORIES); writer.flush(); logger.info("Finished processing {} records", count); } diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java index d3c03c4e2..44ac1c5fb 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java @@ -46,6 +46,7 @@ package fiftyone.ipintelligence.examples.console; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.common.testhelpers.LogbackHelper; import fiftyone.ipintelligence.IPIntelligenceOnPremisePipelineBuilder; import fiftyone.ipintelligence.IPIntelligencePipelineBuilder; @@ -81,6 +82,12 @@ *

    */ public class PerformanceBenchmark { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-ipintelligence-examples-console-performancebenchmark.java"; + // the default number of threads if one is not provided. public static final int DEFAULT_NUMBER_OF_THREADS = 4; // the number of tests to execute. @@ -132,6 +139,8 @@ public static void main(String[] args) throws Exception { evidenceFilename, numberOfThreads, new PrintWriter(System.out,true)); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); } /** @@ -234,7 +243,9 @@ private void setPipelinePerformanceProperties( builder.setPerformanceProfile(config.profile) // set this to false for testing .setAutoUpdate(false) - // set this to false for testing + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. .setShareUsage(false) // hint for cache concurrency .setConcurrency(numberOfThreads); diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java index 8f0e5e8f2..fb84c3f1d 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java @@ -45,6 +45,7 @@ package fiftyone.ipintelligence.examples.console; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.ipintelligence.IPIntelligencePipelineBuilder; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; import fiftyone.ipintelligence.examples.shared.DataFileHelper; @@ -80,6 +81,12 @@ * in production without further testing and tuning. */ public class SuspiciousOnPrem { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java"; + private static final Logger logger = LoggerFactory.getLogger(SuspiciousOnPrem.class); /* In this example, by default, the free 51degrees Lite IP Intelligence data file is used from @@ -116,6 +123,8 @@ public static void main(String[] args) throws Exception { // prepare 'evidence' for use in pipeline (see below) List> evidence = EvidenceHelper.setUpEvidence(); run(dataFile, evidence, System.out); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); } /** @@ -166,7 +175,9 @@ public static void run(String dataFile, // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=automatic-datafile-updates // https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=usage-sharing .setPerformanceProfile(Constants.PerformanceProfiles.LowMemory) - // inhibit sharing usage for this example, usually this should be set to "true" + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. .setShareUsage(false) // inhibit auto-update of the data file for this example .setAutoUpdate(false) diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java index a57a9b3e0..1e3f53ffa 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java @@ -22,6 +22,7 @@ package fiftyone.ipintelligence.examples.console; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.ipintelligence.IPIntelligencePipelineBuilder; import fiftyone.ipintelligence.examples.shared.DataFileHelper; import fiftyone.ipintelligence.shared.testhelpers.KeyUtils; @@ -38,6 +39,7 @@ import java.io.PrintWriter; import java.nio.file.*; import java.text.SimpleDateFormat; +import java.util.Locale; import java.util.Objects; import java.util.Scanner; @@ -178,7 +180,13 @@ */ public class UpdateDataFile { - public static final String UPDATE_EXAMPLE_LICENSE_KEY_NAME = "LicenseKey"; + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "console-src-main-java-fiftyone-ipintelligence-examples-console-updatedatafile.java"; + + public static final String UPDATE_EXAMPLE_LICENSE_KEY_NAME = "_51DEGREES_LICENSE_KEY"; public static final String DEFAULT_DATA_FILENAME = System.getProperty("user.dir") + File.separator + ENTERPRISE_DATA_FILE_REL_PATH; @@ -198,6 +206,8 @@ public static void main(String[] args) throws Exception { String licenseKey = args.length > 0 ? args[0] : null; String dataFilename = args.length > 1 ? args[1] : null; run(dataFilename, licenseKey, true); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); } /** @@ -266,7 +276,7 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr "in your licensed allocation."); System.out.print("Do you wish to continue with this example (y)? "); String input = scanner.nextLine(); - if ((input.isEmpty() || input.toLowerCase().startsWith("y")) == false) { + if ((input.isEmpty() || input.toLowerCase(Locale.ROOT).startsWith("y")) == false) { logger.info("Stopping example without download"); return; } @@ -294,6 +304,10 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr // overwritten the pipeline must be configured to copy the supplied // file to a temporary file (createTempDataCopy parameter == true. .useOnPremise(dataFilename, true) + // Usage sharing is off because this example's evidence is already + // known to 51Degrees. In production, leave usage sharing on, which adds + // a Usage Sharing Element to the Pipeline and helps improve the service. + .setShareUsage(false) // Use the low memory profile, because this example // downloads the Enterprise data file, which is over 6 GB, // and MaxPerformance would load all of it into memory. diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/CloudExamplesWithoutKeyTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/CloudExamplesWithoutKeyTest.java new file mode 100644 index 000000000..d6b4e7e56 --- /dev/null +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/CloudExamplesWithoutKeyTest.java @@ -0,0 +1,82 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.console; + +import fiftyone.ipintelligence.examples.shared.ExampleProcess; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Runs each IP Intelligence cloud console example as it runs from a fresh + * clone, with no resource key anywhere, and checks that it ends without an + * error, names the variable to set and links to where a key is created, + * then ends with its Find out more section. + */ +public class CloudExamplesWithoutKeyTest { + + private static final String CONFIGURE = "https://51degrees.com/configure/"; + + private static final String TAGS = + "?utm_source=code&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content="; + + private static void runWithoutKey(Class example) throws Exception { + ExampleProcess.Result result = ExampleProcess.runWithoutKeys(example); + System.out.println(result.output); + String output = result.output; + assertEquals(example.getSimpleName() + " should end with exit code 0", + 0, result.exitCode); + assertTrue("The output should name _51DEGREES_RESOURCE_KEY", + output.contains("Set the _51DEGREES_RESOURCE_KEY " + + "environment variable")); + int link = output.indexOf(CONFIGURE); + assertTrue("The output should link to where a key is created", + link >= 0 && output.startsWith(TAGS, + output.indexOf('?', link))); + assertTrue("The output should end with the Find out more section " + + "after the link", + output.lastIndexOf("Find out more") > link); + } + + @Test + public void gettingStartedCloud() throws Exception { + runWithoutKey(GettingStartedCloud.class); + } + + @Test + public void gettingStartedCloudMixed() throws Exception { + runWithoutKey(GettingStartedCloudMixed.class); + } + + @Test + public void getAllPropertiesCloud() throws Exception { + runWithoutKey(GetAllPropertiesCloud.class); + } + + @Test + public void metadataCloud() throws Exception { + runWithoutKey(MetadataCloud.class); + } +} diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/CsvNumbersTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/CsvNumbersTest.java new file mode 100644 index 000000000..90bb4e06b --- /dev/null +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/CsvNumbersTest.java @@ -0,0 +1,49 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.console; + +import org.junit.Test; + +import java.util.Locale; + +import static org.junit.Assert.assertEquals; + +/** + * The numbers CompareOnPrem and MetricsOnPrem write into their CSV files + * have a full stop as the decimal separator whatever the default locale, + * because a decimal comma would split one column into two. + */ +public class CsvNumbersTest { + + @Test + public void csvNumbers_GermanDefaultLocale_FullStop() { + Locale saved = Locale.getDefault(); + Locale.setDefault(Locale.GERMANY); + try { + assertEquals("12.346", CompareOnPrem.kilometres(12.3456)); + assertEquals("3.50", MetricsOnPrem.twoDecimalPlaces(3.5)); + } finally { + Locale.setDefault(saved); + } + } +} diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/ExamplesWithoutSettingsTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/ExamplesWithoutSettingsTest.java index d14a522f6..8f6bf1417 100644 --- a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/ExamplesWithoutSettingsTest.java +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/ExamplesWithoutSettingsTest.java @@ -23,6 +23,7 @@ package fiftyone.ipintelligence.examples.console; import fiftyone.ipintelligence.examples.shared.DataFileHelper; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import org.junit.Test; import java.io.ByteArrayOutputStream; @@ -47,6 +48,11 @@ public class ExamplesWithoutSettingsTest { private static final String LICENSE_KEY = UpdateDataFile.UPDATE_EXAMPLE_LICENSE_KEY_NAME; + /** + * A name that is not a performance profile. + */ + private static final String UNKNOWN_PROFILE = "NoSuchProfile"; + /** * An example's main method. */ @@ -55,43 +61,35 @@ private interface Main { } /** - * Runs the example's main with no arguments and the data file and - * licence key system properties removed, and returns what it wrote to - * standard output. The deprecated name of the data file variable is - * read when the current one is not set, so both are checked and - * removed. + * Runs the example's main with the arguments given, none unless a test + * gives some, and the data file and licence key system properties + * removed, and returns what it wrote to standard output. */ - @SuppressWarnings("deprecation") - private static String runWithoutSettings(Main main) throws Exception { + private static String runWithoutSettings(Main main, String... args) + throws Exception { assumeTrue("Skipping, a data file or licence key is set in the environment", System.getenv(DataFileHelper.IPI_PATH_VARIABLE) == null && - System.getenv(DataFileHelper.IPI_PATH_ENV_VAR) == null && System.getenv(LICENSE_KEY) == null && System.getenv(LICENSE_KEY.toUpperCase()) == null); String dataFile = System.clearProperty(DataFileHelper.IPI_PATH_VARIABLE); - String oldNameDataFile = - System.clearProperty(DataFileHelper.IPI_PATH_ENV_VAR); String licenseKey = System.clearProperty(LICENSE_KEY); PrintStream out = System.out; ByteArrayOutputStream buffer = new ByteArrayOutputStream(); System.setOut(new PrintStream(buffer, true, "UTF-8")); try { - main.main(new String[0]); + main.main(args); } finally { System.setOut(out); if (dataFile != null) { System.setProperty(DataFileHelper.IPI_PATH_VARIABLE, dataFile); } - if (oldNameDataFile != null) { - System.setProperty(DataFileHelper.IPI_PATH_ENV_VAR, - oldNameDataFile); - } if (licenseKey != null) { System.setProperty(LICENSE_KEY, licenseKey); } } String output = buffer.toString("UTF-8"); out.println(output); + assertFindOutMore(output); return output; } @@ -100,6 +98,21 @@ private static void assertContains(String output, String expected) { output.contains(expected)); } + /** + * Checks that the output has the "Find out more" section after the last + * link to the contact page, and that the section closes with its rule. + * Log lines may still follow the section. + */ + private static void assertFindOutMore(String output) { + int heading = output.lastIndexOf(FindOutMore.HEADING); + assertTrue("The output should have the '" + FindOutMore.HEADING + + "' section after the link to the contact page", + heading > output.lastIndexOf(CONTACT_US)); + assertTrue("The '" + FindOutMore.HEADING + + "' section should end with its rule", + output.indexOf(FindOutMore.RULE, heading) > heading); + } + private static void assertStops(String output, String purpose) { assertContains(output, purpose + " needs the "); assertContains(output, "so the example stops here."); @@ -138,6 +151,36 @@ public void metrics() throws Exception { "Measuring the location areas"); } + /** + * A name that is not a performance profile ends the example after the + * names it accepts, and the section still follows. + */ + @Test + public void compare_UnknownProfile() throws Exception { + assertContains( + runWithoutSettings(CompareOnPrem::main, + DataFileHelper.LITE_DATA_FILE_REL_PATH, + CompareOnPrem.GEOIP_COMPARISON_EVIDENCE_REL_PATH, + CompareOnPrem.DEFAULT_OUTPUT_FILE, + UNKNOWN_PROFILE), + "'" + UNKNOWN_PROFILE + "' is not a performance profile."); + } + + /** + * A name that is not a performance profile ends the example after the + * names it accepts, and the section still follows. + */ + @Test + public void metrics_UnknownProfile() throws Exception { + assertContains( + runWithoutSettings(MetricsOnPrem::main, + DataFileHelper.LITE_DATA_FILE_REL_PATH, + MetricsOnPrem.DEFAULT_OUTPUT_FILE, + String.valueOf(MetricsOnPrem.DEFAULT_SAMPLE_PERCENTAGE), + UNKNOWN_PROFILE), + "'" + UNKNOWN_PROFILE + "' is not a performance profile."); + } + @Test public void offlineProcessing() throws Exception { String output = runWithoutSettings(OfflineProcessing::main); diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/FindOutMoreCoverageTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/FindOutMoreCoverageTest.java new file mode 100644 index 000000000..3ad6721cf --- /dev/null +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/FindOutMoreCoverageTest.java @@ -0,0 +1,74 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.console; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertTrue; + +/** + * Checks that every console example with a main method writes the "Find out + * more" section, including those that need a key or a paid data file and so + * do not run in ExamplesWithoutSettingsTest. + */ +public class FindOutMoreCoverageTest { + + @Test + public void everyExample_WritesFindOutMore() throws IOException { + Path sources = Paths.get(System.getProperty("basedir", "."), + "src", "main", "java"); + assertTrue(sources + " should hold the examples", + Files.isDirectory(sources)); + List files; + try (Stream walk = Files.walk(sources)) { + files = walk.filter(f -> f.toString().endsWith(".java")) + .collect(Collectors.toList()); + } + List examples = new ArrayList<>(); + List missing = new ArrayList<>(); + for (Path file : files) { + String text = new String(Files.readAllBytes(file), + StandardCharsets.UTF_8); + if (text.contains("static void main(")) { + examples.add(file.getFileName().toString()); + if (text.contains("FindOutMore.write") == false) { + missing.add(file.getFileName().toString()); + } + } + } + assertTrue("No examples were found in " + sources, + examples.isEmpty() == false); + assertTrue("These examples do not write the 'Find out more' " + + "section: " + missing, missing.isEmpty()); + } +} diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/UpdateDataFileTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/UpdateDataFileTest.java index 06a29ba6d..5a7164325 100644 --- a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/UpdateDataFileTest.java +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/UpdateDataFileTest.java @@ -35,7 +35,9 @@ public class UpdateDataFileTest { @Test public void updateExampleTest () throws Exception { String licenseKey = KeyUtils.getNamedKey(UpdateDataFile.UPDATE_EXAMPLE_LICENSE_KEY_NAME); - assumeFalse("Skipping test, no license key found", + assumeFalse("Skipped, because " + + UpdateDataFile.UPDATE_EXAMPLE_LICENSE_KEY_NAME + + " is not set to a license key for the data files.", KeyUtils.isInvalidKey(licenseKey)); try { run(null, licenseKey, false); diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/UsageSharingCoverageTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/UsageSharingCoverageTest.java new file mode 100644 index 000000000..997a71517 --- /dev/null +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/UsageSharingCoverageTest.java @@ -0,0 +1,166 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.console; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertTrue; + +/** + * Checks that no console example adds a Usage Sharing Element to its + * Pipeline, and that each says beside the setting that turns it off that + * usage sharing should be on in production. The builders that add the + * element unless told not to are the 51Degrees Pipeline builder and the + * on-premise IP Intelligence Pipeline builder. + */ +public class UsageSharingCoverageTest { + + /** + * Each builder that adds a Usage Sharing Element unless it is turned + * off. + */ + private static final Pattern SHARING_BUILDER = Pattern.compile( + "new FiftyOnePipelineBuilder\\(|\\.useOnPremise\\("); + + private static final Pattern SHARING_OFF = Pattern.compile( + "\\.setShareUsage\\(false\\)"); + + /** + * The words the comment before each setting that turns usage sharing + * off must hold. + */ + private static final String PRODUCTION_COMMENT = + "In production, leave usage sharing on"; + + /** + * How many lines before the setting the comment may start. + */ + private static final int COMMENT_LINES = 6; + + @Test + public void everyExample_TurnsUsageSharingOff() throws IOException { + List examples = new ArrayList<>(); + List sharing = new ArrayList<>(); + for (Path file : exampleSources()) { + String text = code(read(file)); + examples.add(file.getFileName().toString()); + int builders = count(SHARING_BUILDER, text); + int off = count(SHARING_OFF, text); + if (off < builders) { + sharing.add(file.getFileName() + " (" + builders + + " builders, " + off + " turned off)"); + } + } + assertTrue("No examples were found", examples.isEmpty() == false); + assertTrue("These console examples share usage: " + sharing, + sharing.isEmpty()); + } + + @Test + public void everySettingOff_SaysToShareInProduction() throws IOException { + List missing = new ArrayList<>(); + for (Path file : exampleSources()) { + String[] lines = read(file).split("\n"); + for (int i = 0; i < lines.length; i++) { + if (SHARING_OFF.matcher(lines[i]).find()) { + StringBuilder before = new StringBuilder(); + for (int j = Math.max(0, i - COMMENT_LINES); j < i; j++) { + before.append(lines[j]).append(' '); + } + String comment = before.toString() + .replaceAll("\\s*(//|\\*)\\s*", " ") + .replaceAll("\\s+", " "); + if (comment.contains(PRODUCTION_COMMENT) == false) { + missing.add(file.getFileName() + ":" + (i + 1)); + } + } + } + } + assertTrue("These settings that turn usage sharing off do not say " + + "to leave it on in production: " + missing, missing.isEmpty()); + } + + /** + * The source of every console example, being each file with a main + * method. + */ + private static List exampleSources() throws IOException { + Path sources = Paths.get(System.getProperty("basedir", "."), + "src", "main", "java"); + assertTrue(sources + " should hold the examples", + Files.isDirectory(sources)); + List examples = new ArrayList<>(); + try (Stream walk = Files.walk(sources)) { + for (Path file : walk + .filter(f -> f.toString().endsWith(".java")) + .collect(Collectors.toList())) { + if (read(file).contains("static void main(")) { + examples.add(file); + } + } + } + return examples; + } + + /** + * The lines of a source that are code, leaving out the comment lines, + * which can show a builder in an example of its use. + */ + private static String code(String text) { + StringBuilder code = new StringBuilder(); + for (String line : text.split("\n")) { + String trimmed = line.trim(); + if (trimmed.startsWith("//") == false && + trimmed.startsWith("*") == false && + trimmed.startsWith("/*") == false) { + code.append(line).append('\n'); + } + } + return code.toString(); + } + + private static String read(Path file) throws IOException { + return new String(Files.readAllBytes(file), StandardCharsets.UTF_8); + } + + private static int count(Pattern pattern, String text) { + Matcher matcher = pattern.matcher(text); + int count = 0; + while (matcher.find()) { + count++; + } + return count; + } +} diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/WeightingShownTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/WeightingShownTest.java new file mode 100644 index 000000000..d6d762a58 --- /dev/null +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/WeightingShownTest.java @@ -0,0 +1,70 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.console; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.Assert.assertTrue; + +/** + * "Examples MUST demonstrate accessing both the value and its associated + * weighting" (ip-intelligence-specification/required-examples.md line 107). + * The getting started examples are where a developer first reads how to get + * a result, so their own code reads each value with its weighting, rather + * than leaving it to code in another file the developer does not see. + */ +public class WeightingShownTest { + + private static String source(String name) throws IOException { + Path path = Paths.get("src", "main", "java", "fiftyone", + "ipintelligence", "examples", "console", name + ".java"); + return new String(Files.readAllBytes(path), StandardCharsets.UTF_8); + } + + private static void assertShowsWeighting(String name) throws IOException { + String source = source(name); + assertTrue(name + " does not read a weighting", + source.contains(".getWeighting()")); + assertTrue(name + " does not read the weighted value", + source.contains("weightedValues(") && + source.contains(".getValue()")); + } + + @Test + public void gettingStartedOnPrem_ShowsValueAndWeighting() + throws IOException { + assertShowsWeighting("GettingStartedOnPrem"); + } + + @Test + public void gettingStartedCloud_ShowsValueAndWeighting() + throws IOException { + assertShowsWeighting("GettingStartedCloud"); + } +} diff --git a/examples/ip-intelligence/pom.xml b/examples/ip-intelligence/pom.xml index 8d7ff14cf..d2cfea059 100644 --- a/examples/ip-intelligence/pom.xml +++ b/examples/ip-intelligence/pom.xml @@ -63,7 +63,7 @@ 2.0.20 1.3.16 - 3.26.0 + 3.28.0 @@ -152,10 +152,11 @@ ${maven.multiModuleProjectDirectory} - - ${TestResourceKey} - - ${LicenseKey} + + <_51DEGREES_RESOURCE_KEY>${_51DEGREES_RESOURCE_KEY} + <_51DEGREES_LICENSE_KEY>${_51DEGREES_LICENSE_KEY} diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java index e617da672..e85ad6b37 100644 --- a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java @@ -47,7 +47,6 @@ import java.util.List; import java.util.Locale; import java.util.Objects; -import java.util.concurrent.atomic.AtomicBoolean; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -137,17 +136,13 @@ public static String findAvailableDataFile(String... candidates) { public static final String IPI_PATH_VARIABLE = "_51DEGREES_IPI_PATH"; /** - * The old name of {@link #IPI_PATH_VARIABLE}, read only when that one - * is not set. + * The old name of {@link #IPI_PATH_VARIABLE}. It is no longer read. * @deprecated use {@link #IPI_PATH_VARIABLE}, and set the variable it * names in place of this one. */ @Deprecated public static final String IPI_PATH_ENV_VAR = "51DEGREES_IPI_PATH"; - // Set once the use of the deprecated name has been logged. - private static final AtomicBoolean OLD_NAME_LOGGED = - new AtomicBoolean(false); /** * The value of the environment variable of this name, or of the system @@ -420,9 +415,7 @@ public static File getEvidenceFile(String evidenceFilename) throws Exception { * An absolute path supplied by the caller is used as is. Otherwise an * explicit path supplied via the {@link #IPI_PATH_VARIABLE} environment * variable or system property is checked before the folder hierarchy is - * searched for the passed filename. When neither is set the deprecated - * {@link #IPI_PATH_ENV_VAR} is read, and its first use is logged as a - * warning. + * searched for the passed filename. * @param dataFilename a filename to find (can be absolute or relative path) * @return a full pathname * @throws Exception if the file was not found @@ -435,19 +428,6 @@ public static String getDataFileLocation(String dataFilename) throws Exception { Paths.get(dataFilename).isAbsolute() == false) { String envName = IPI_PATH_VARIABLE; String envDataFile = environmentOrProperty(envName); - if (Objects.isNull(envDataFile)) { - envName = IPI_PATH_ENV_VAR; - envDataFile = environmentOrProperty(envName); - if (Objects.nonNull(envDataFile) - && OLD_NAME_LOGGED.compareAndSet(false, true)) { - logger.warn( - "The environment variable " + IPI_PATH_ENV_VAR - + " is deprecated and " + IPI_PATH_VARIABLE - + " replaces it. Set " + IPI_PATH_VARIABLE - + " to the same value, as a later release may no longer " - + "read " + IPI_PATH_ENV_VAR + "."); - } - } if (Objects.nonNull(envDataFile)) { if (Files.exists(Paths.get(envDataFile))) { return envDataFile; diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/ExampleProcess.java b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/ExampleProcess.java new file mode 100644 index 000000000..e411b3e3d --- /dev/null +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/ExampleProcess.java @@ -0,0 +1,127 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.shared; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +/** + * Runs an example's main method in a new Java process with no key in its + * environment, as someone runs it from a fresh clone, for the tests of + * what an example does with no settings. A new process is needed because + * a running Java process cannot remove a variable from its own + * environment. + */ +public final class ExampleProcess { + + /** + * How long an example may run before the test stops it. + */ + private static final long TIMEOUT_SECONDS = 120; + + /** + * What an example wrote and how it ended. + */ + public static final class Result { + /** + * The process's exit code. + */ + public final int exitCode; + + /** + * Everything the process wrote to standard output and standard + * error. + */ + public final String output; + + Result(int exitCode, String output) { + this.exitCode = exitCode; + this.output = output; + } + } + + private ExampleProcess() { + } + + /** + * Runs the example's main method with no arguments and with every + * environment variable whose name holds "KEY", in any case, left out. + * The process takes the class path of the process that starts it, and + * of its system properties only project.root, which says where the + * files the example reads are. + * @param example the class whose main method runs + * @return the exit code and the output + * @throws Exception if the process cannot start, or does not end + * within the timeout + */ + public static Result runWithoutKeys(Class example) throws Exception { + List command = new ArrayList<>(); + command.add(new File(new File(System.getProperty("java.home"), + "bin"), "java").getPath()); + command.add("-cp"); + command.add(System.getProperty("java.class.path")); + String projectRoot = System.getProperty("project.root"); + if (projectRoot != null) { + command.add("-Dproject.root=" + projectRoot); + } + command.add(example.getName()); + + ProcessBuilder builder = new ProcessBuilder(command); + builder.environment().keySet().removeIf( + name -> name.toUpperCase(Locale.ROOT).contains("KEY")); + builder.redirectErrorStream(true); + Process process = builder.start(); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + Thread reader = new Thread(() -> copy(process.getInputStream(), + output)); + reader.start(); + if (process.waitFor(TIMEOUT_SECONDS, TimeUnit.SECONDS) == false) { + process.destroyForcibly(); + reader.join(); + throw new AssertionError(example.getName() + + " did not end within " + TIMEOUT_SECONDS + " seconds. " + + "It wrote:\n" + output.toString("UTF-8")); + } + reader.join(); + return new Result(process.exitValue(), output.toString("UTF-8")); + } + + private static void copy(InputStream input, ByteArrayOutputStream output) { + byte[] buffer = new byte[8192]; + try { + int read; + while ((read = input.read(buffer)) >= 0) { + output.write(buffer, 0, read); + } + } catch (IOException e) { + // The process ended, so there is no more to read. + } + } +} diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/FindOutMore.java b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/FindOutMore.java new file mode 100644 index 000000000..f893c41c3 --- /dev/null +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/FindOutMore.java @@ -0,0 +1,200 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.shared; + +import java.io.OutputStream; +import java.io.PrintWriter; +import java.util.regex.Pattern; + +/** + * Writes the "Find out more" section that ends every example's output, set + * apart from the example's own output by a rule above and below. It gives + * pages on 51degrees.com, each tagged with the example that wrote it, then + * the repositories behind the example. + */ +public final class FindOutMore { + + /** + * The heading the section starts with. + */ + public static final String HEADING = "Find out more"; + + /** + * The rule above and below the section. + */ + public static final String RULE = + "----------------------------------------------------------------"; + + /** + * The repositories an on-premise example lists, being this SDK, the C + * engines it builds on and the free data files. + */ + public static final String[] ON_PREMISE_REPOSITORIES = { + "https://github.com/51Degrees/51degrees-java", + "https://github.com/51Degrees/51degrees-cxx", + "https://github.com/51Degrees/data" + }; + + /** + * The repositories a cloud example lists. + */ + public static final String[] CLOUD_REPOSITORIES = { + "https://github.com/51Degrees/51degrees-java" + }; + + /** + * Each page's label and its path on 51degrees.com. + */ + private static final String[][] PAGES = { + {"IP Intelligence", "ip-intelligence"}, + {"Pricing", "pricing"}, + {"Documentation", "documentation/index.html"} + }; + + /** + * What a campaign tag value may hold. + */ + private static final Pattern SLUG = + Pattern.compile("[a-z0-9][a-z0-9._-]*"); + + private FindOutMore() { + } + + /** + * Writes the section. + * @param output where to write, flushed afterwards + * @param example the example's tag, a lower case slug such as + * "console-gettingstartedonprem.java" + * @param repositories the repositories to list, such as + * {@link #ON_PREMISE_REPOSITORIES} + */ + public static void write( + PrintWriter output, + String example, + String... repositories) { + output.println(); + writeLines(output, "", example, repositories); + } + + /** + * Writes the section with each line a comment, for output that has to + * stay valid in its own format, such as a YAML stream. + * @param output where to write, flushed afterwards + * @param comment what starts a comment line, such as "# " + * @param example the example's tag + * @param repositories the repositories to list + */ + public static void writeCommented( + PrintWriter output, + String comment, + String example, + String... repositories) { + writeLines(output, comment, example, repositories); + } + + private static void writeLines( + PrintWriter output, + String start, + String example, + String... repositories) { + output.println(start + RULE); + output.println(start + HEADING); + for (String[] page : PAGES) { + output.println(start + "\t" + page[0] + ": " + + link(page[1], example)); + } + for (String repository : repositories) { + output.println(start + "\t" + repository); + } + output.println(start + RULE); + output.flush(); + } + + /** + * Writes the section to a stream. See + * {@link #write(PrintWriter, String, String...)}. + */ + public static void write( + OutputStream output, + String example, + String... repositories) { + write(new PrintWriter(output), example, repositories); + } + + /** + * The section as HTML, for the end of a web example's page. + * @param example the example's tag + * @param repositories the repositories to list + * @return the markup + */ + public static String html(String example, String... repositories) { + StringBuilder html = new StringBuilder(); + html.append("

    \n"); + html.append("

    ").append(HEADING).append("

    \n"); + html.append(" \n
    \n"); + return html.toString(); + } + + /** + * The address of a page on 51degrees.com, tagged with the example. + * @param page the path of the page + * @param example the example's tag + * @return the address + * @throws IllegalArgumentException if the tag is not a lower case slug + */ + public static String link(String page, String example) { + return link(page, example, "find-out-more"); + } + + /** + * The address of a page on 51degrees.com, tagged with the example and + * the part of its output that gives the link. + * @param page the path of the page + * @param example the example's tag + * @param term the part of the output, a lower case slug + * @return the address + * @throws IllegalArgumentException if a tag is not a lower case slug + */ + public static String link(String page, String example, String term) { + for (String tag : new String[]{example, term}) { + if (tag == null || SLUG.matcher(tag).matches() == false) { + throw new IllegalArgumentException( + "'" + tag + "' is not a lower case slug."); + } + } + return "https://51degrees.com/" + + page + "?utm_source=code&utm_medium=example" + + "&utm_campaign=51degrees-java&utm_content=" + example + + "&utm_term=" + term; + } +} diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java index 2342f4047..b7604c485 100644 --- a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java @@ -26,109 +26,137 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.PrintStream; import java.util.*; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; public class KeyHelper { /** - * Aligned name of the environment variable or system property which may - * hold the resource key. This name is checked first, before the legacy - * {@link #TEST_RESOURCE_KEY} name. + * Name of the environment variable or system property which holds the + * resource key. It is the only name read. */ public static final String RESOURCE_KEY_VARIABLE = "_51DEGREES_RESOURCE_KEY"; /** - * The old name of {@link #RESOURCE_KEY_VARIABLE}, read only when that - * one is not set. + * The old name of {@link #RESOURCE_KEY_VARIABLE}. It is no longer read. * @deprecated use {@link #RESOURCE_KEY_VARIABLE}, and set the variable * it names in place of this one. */ @Deprecated public static final String RESOURCE_KEY_ENV_VAR = "51DEGREES_RESOURCE_KEY"; - // Set once the use of the deprecated name has been logged. - private static final AtomicBoolean OLD_NAME_LOGGED = - new AtomicBoolean(false); - public static final String TEST_RESOURCE_KEY = "TestResourceKey"; + /** + * The variable the cloud examples and their tests read a resource key + * from, which is {@link #RESOURCE_KEY_VARIABLE}. + */ + public static final String TEST_RESOURCE_KEY = RESOURCE_KEY_VARIABLE; static Logger logger = LoggerFactory.getLogger(KeyHelper.class); + static final String CREATE_KEY_URL = + "https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-ipintelligence-examples-shared-keyhelper.java&utm_term=no-resource-key"; + /** - * The names every cloud test reads a resource key from first, before - * the names its own example reads. + * The configuration page that creates a key with the free properties + * the cloud examples use. */ - public static final List SHARED_RESOURCE_KEY_NAMES = - Collections.unmodifiableList(Arrays.asList( - "_51DEGREES_RESOURCE_KEY", - "_51DEGREES_RESOURCE_KEY_BESPOKE")); + public static final String FREE_PROPERTIES_CONFIGURATION = + "configure/Wkqxf3Bs"; - static final String CREATE_KEY_URL = - "https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-ipintelligence-examples-shared-keyhelper.java&utm_term=no-resource-key"; + /** + * The resource key a cloud example runs with, being the key given as + * its argument, or else the key in {@link #RESOURCE_KEY_VARIABLE}, read + * as an environment variable, under its upper case name and as a + * system property. + * @param argument the key given on the command line, or null + * @return the key, or null when there is none + */ + public static String findResourceKey(String argument) { + if (KeyUtils.isInvalidKey(argument) == false) { + return argument.trim(); + } + String value = readKey(RESOURCE_KEY_VARIABLE); + return value == null ? null : value.trim(); + } + + /** + * Writes what a cloud example needs when it has no resource key, being + * the variable to set and the page where a key is created, so that the + * example can end without an error. + * @param output where to write + * @param example the example's tag, as its Find out more section has it + */ + public static void writeNoResourceKey(PrintStream output, + String example) { + output.println(); + output.println("This example calls the 51Degrees cloud service, " + + "which needs a resource key, and none is set."); + output.println("Set the " + RESOURCE_KEY_VARIABLE + " environment " + + "variable to a resource key, or give the key as the first " + + "argument, and run the example again."); + output.println("Create a resource key with the properties this " + + "example uses at " + FindOutMore.link( + FREE_PROPERTIES_CONFIGURATION, example, "no-resource-key")); + output.flush(); + } /** * The resource key for a test that calls the live cloud service, from - * {@link #SHARED_RESOURCE_KEY_NAMES} and then the names - * {@link #getOrSetTestResourceKey()} reads. Fails the test when there - * is none, because a cloud test that passes without a key has called - * nothing. + * {@link #RESOURCE_KEY_VARIABLE}, the name + * {@link #getOrSetTestResourceKey()} reads. Skips the test, naming the + * variable, when there is none, because a cloud test that passes + * without a key has called nothing. * @param asksFor what the test asks the service for * @param products the products the key has to carry * @return the resource key */ public static String requireTestResourceKey(String asksFor, String... products) { - return requireResourceKey(KeyHelper::readKey, Arrays.asList( - RESOURCE_KEY_VARIABLE, TEST_RESOURCE_KEY), asksFor, products); + return requireResourceKey(KeyHelper::readKey, + Collections.singletonList(RESOURCE_KEY_VARIABLE), asksFor, + products); } static String requireResourceKey(Function read, List exampleNames, String asksFor, String... products) { - List names = new ArrayList<>(SHARED_RESOURCE_KEY_NAMES); - names.addAll(exampleNames); - for (String name : names) { + for (String name : exampleNames) { String value = read.apply(name); - if (KeyUtils.isInvalidKey(value) - && RESOURCE_KEY_VARIABLE.equals(name)) { - value = oldNameResourceKey(read); - } if (KeyUtils.isInvalidKey(value) == false) { return value.trim(); } } - throw new AssertionError(noResourceKey(names, asksFor, products)); + throw skipTest(noResourceKey(exampleNames, asksFor, products)); } /** - * The key held under the deprecated {@link #RESOURCE_KEY_ENV_VAR}, or - * null when there is none. It is read only when - * {@link #RESOURCE_KEY_VARIABLE} holds no key, and its first use is - * logged as a warning. + * The exception that skips a test, being JUnit's + * AssumptionViolatedException. It is made by name, because the + * examples use KeyHelper too, and run with no test library on their + * class path. Where JUnit cannot be found, the exception is an + * IllegalStateException with the same message. + * @param message why the test is skipped + * @return the exception to throw */ - private static String oldNameResourceKey(Function read) { - String value = read.apply(RESOURCE_KEY_ENV_VAR); - if (KeyUtils.isInvalidKey(value)) { - return null; + static RuntimeException skipTest(String message) { + try { + return (RuntimeException) Class.forName( + "org.junit.AssumptionViolatedException") + .getConstructor(String.class) + .newInstance(message); + } catch (ReflectiveOperationException | LinkageError | + ClassCastException e) { + return new IllegalStateException(message); } - if (OLD_NAME_LOGGED.compareAndSet(false, true)) { - logger.warn( - "The environment variable " + RESOURCE_KEY_ENV_VAR - + " is deprecated and " + RESOURCE_KEY_VARIABLE - + " replaces it. Set " + RESOURCE_KEY_VARIABLE - + " to the same value, as a later release may no longer " - + "read " + RESOURCE_KEY_ENV_VAR + "."); - } - return value; } static String noResourceKey(List names, String asksFor, String... products) { return "This test calls the live 51Degrees cloud service, so it " + - "needs a resource key and there is nothing to fall back on. " + - "None of " + String.join(", ", names) + " is set, as an " + - "environment variable or as a system property, tried in that " + - "order. The key has to carry the " + productList(products) + + "needs a resource key, and it is skipped because " + + String.join(", ", names) + " is not set as an environment " + + "variable or as a system property. " + + "The key has to carry the " + productList(products) + ", because the test asks the service for " + asksFor + ". To " + "see the products a key carries, open " + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=. " + @@ -154,7 +182,7 @@ private static String productList(String... products) { private static String readKey(String name) { for (String value : new String[]{ System.getenv(name), - System.getenv(name.toUpperCase()), + System.getenv(name.toUpperCase(Locale.ROOT)), System.getProperty(name)}) { if (KeyUtils.isInvalidKey(value) == false) { return value; @@ -177,7 +205,7 @@ public static String getOrSetTestResourceKey(boolean shouldThrow) { public static String getOrSetTestResourceKey(String value, boolean shouldThrow) { return getOrSetResourceKey(value, TEST_RESOURCE_KEY, "A free resource key may be obtained from " + - "https://configure.51degrees.com/Wkqxf3Bs?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-ipintelligence-examples-shared-keyhelper.java&utm_term=resource-key-required. A free key " + + "https://51degrees.com/configure/Wkqxf3Bs?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-ipintelligence-examples-shared-keyhelper.java&utm_term=resource-key-required. A free key " + "populates the free tier properties only. See " + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-ipintelligence-examples-shared-keyhelper.java&utm_term=resource-key-required to get a paid subscription " + "with more properties.", @@ -187,36 +215,23 @@ public static String getOrSetTestResourceKey(String value) { return getOrSetTestResourceKey(value, true); } /** - * Obtain a resource key from the passed argument, - * from environment variable or from a property. The aligned - * {@link #RESOURCE_KEY_VARIABLE} name is checked first, then the - * deprecated {@link #RESOURCE_KEY_ENV_VAR}, then the passed variable - * name. Store as System Property under the passed variable name. + * Obtain a resource key from the passed argument, or else from the + * passed variable name, as an environment variable or a system + * property. Store as System Property under the passed variable name. */ public static String getOrSetResourceKey(String value, String variableName, String errorMessage, boolean shouldThrow) { if (Objects.isNull(value)) { - // check the aligned name first, then the legacy name - value = KeyUtils.getNamedKey(RESOURCE_KEY_VARIABLE); - if (Objects.isNull(value) || value.isEmpty()) { - String oldNameKey = oldNameResourceKey(KeyUtils::getNamedKey); - if (Objects.nonNull(oldNameKey)) { - value = oldNameKey; - } - } - if (Objects.isNull(value)) { - value = KeyUtils.getNamedKey(variableName); - } + value = KeyUtils.getNamedKey(variableName); } if (KeyUtils.isInvalidKey(value)) { logger.error("\nTo access Cloud Services you must supply a " + "\"ResourceKey\" in one of the following ways: \n - in the " + "configuration file of an example,\n - as a command line parameter of a " + "runnable example,\n - as an Environment Variable named \"\u001B[36m{}\u001B[0m\"," + - "\n - as a System Property named \"\u001B[36m{}\u001B[0m\" " + - "(the legacy name \"\u001B[36m{}\u001B[0m\" is also still accepted).", - RESOURCE_KEY_VARIABLE, RESOURCE_KEY_VARIABLE, variableName); + "\n - as a System Property named \"\u001B[36m{}\u001B[0m\").", + variableName, variableName); logger.error(errorMessage); if (shouldThrow) { throw new IllegalStateException("\"" + value + "\" is not a valid resource key"); @@ -240,7 +255,7 @@ public static String getOrSetSuperResourceKey(String value, String variablename, "with more properties. " + "Once subscribed, a resource key with the properties required " + "by this example can be created at " + - "https://configure.51degrees.com/hYzn3TV3?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-ipintelligence-examples-shared-keyhelper.java&utm_term=super-resource-key-required.", + "https://51degrees.com/configure/hYzn3TV3?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-ipintelligence-examples-shared-keyhelper.java&utm_term=super-resource-key-required.", shouldThrow); } } diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/PropertyHelper.java b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/PropertyHelper.java index 68b45b83b..41ba0c5cc 100644 --- a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/PropertyHelper.java +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/PropertyHelper.java @@ -23,6 +23,7 @@ package fiftyone.ipintelligence.examples.shared; import fiftyone.pipeline.core.data.IWeightedValue; +import fiftyone.pipeline.core.data.WeightedValue; import fiftyone.pipeline.core.data.WktString; import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; @@ -30,8 +31,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.net.InetAddress; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -45,9 +49,10 @@ public class PropertyHelper { * property. */ public static final String PRICING_MESSAGE = "Some properties used " + - "by this example are not available with a free resource key. " + - "See https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-ipintelligence-examples-shared-propertyhelper.java&utm_term=pricing_message to get a paid subscription " + - "with more properties."; + "by this example are not available with a free resource key or " + + "the free data file. Contact us about a key or a data file " + + "that has them at " + + "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-ipintelligence-examples-shared-propertyhelper.java&utm_term=paid-properties"; static Logger logger = LoggerFactory.getLogger(PropertyHelper.class); @@ -58,8 +63,8 @@ public class PropertyHelper { * Unwrap a value that may be a List of IWeightedValue at runtime * (due to Java generics type erasure, the on-premise engine may return * List<IWeightedValue<T>> even when the declared type is T). - * For single-element lists, returns just the value. - * For multi-element lists, returns values joined with ", ". + * A list's values are joined with ", ", and a weighted value is + * followed by its weighting, as in "Japan (weighting 1.00)". * @param value the raw value from AspectPropertyValue.getValue() * @return the unwrapped string representation */ @@ -73,14 +78,36 @@ private static String unwrapValue(Object value) { return list.stream() .map(item -> { if (item instanceof IWeightedValue) { - Object inner = ((IWeightedValue) item).getValue(); - return inner != null ? inner.toString() : "Unknown"; + // An IP Intelligence value comes with the + // weighting the data gives it, which is 1 for + // a property with one answer. + IWeightedValue weighted = (IWeightedValue) item; + return asWeightedString( + weighted.getValue(), + weighted.getWeighting()); } - return item != null ? item.toString() : "Unknown"; + return toDisplayString(item); }) .collect(Collectors.joining(", ")); } - return value != null ? value.toString() : "Unknown"; + return toDisplayString(value); + } + + /** + * Text for a single value. An address is shown in its literal form, for + * example "8.8.8.0", because InetAddress.toString() puts the host name + * and a slash in front of it. + * @param value a single unwrapped value, may be null + * @return the text to display + */ + private static String toDisplayString(Object value) { + if (value == null) { + return "Unknown"; + } + if (value instanceof InetAddress) { + return ((InetAddress) value).getHostAddress(); + } + return value.toString(); } /** * Build the "no value" output for a property and log the @@ -111,8 +138,9 @@ public static AspectPropertyValue tryGet(Supplier> } catch (PropertyMissingException e) { String message = "The property '" + e.getPropertyName() + "' is not " + - "available in this data file. See data file options " + - "here"; + "available with the data file or resource key in use. " + + "Contact us about one that has it at " + + "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-ipintelligence-examples-shared-propertyhelper.java&utm_term=property-missing"; AspectPropertyValue result = new AspectPropertyValueDefault<>(); result.setNoValueMessage(message); return result; @@ -158,6 +186,7 @@ public static String asFloatProperty(AspectPropertyValue property) { /** * Helper to get the value of an IP Intelligence InetAddress property. * Handles runtime List<IWeightedValue> values due to type erasure. + * The address is shown in its literal form, for example "8.8.8.0". */ public static String asIPAddressProperty(AspectPropertyValue property) { if (property == null || !property.hasValue()) { @@ -193,6 +222,75 @@ public static String asString(AspectPropertyValue value) { return "Unknown. " + value.getNoValueMessage(); } + /** + * The raw weighting of a value that is the only answer, which is a + * weighting of 1. + */ + private static final int FULL_WEIGHTING = 65535; + + /** + * Get each value of an IP Intelligence property with its weighting. IP + * Intelligence properties return weighted values, and a property with a + * single answer returns it with a weighting of 1. Due to Java generics + * type erasure, the on-premise engine returns + * List<IWeightedValue<T>> even when the declared type is T, so + * a value that is not such a list is given here with a weighting of 1. + * @param property the property value, may be null + * @return the weighted values, or an empty list when there is no value + */ + public static List> weightedValues( + AspectPropertyValue property) { + List> result = new ArrayList<>(); + if (property == null || property.hasValue() == false) { + return result; + } + Object value = property.getValue(); + if (value instanceof List) { + for (Object item : (List) value) { + if (item instanceof IWeightedValue) { + result.add((IWeightedValue) item); + } else { + result.add(new WeightedValue<>(FULL_WEIGHTING, item)); + } + } + } else { + result.add(new WeightedValue<>(FULL_WEIGHTING, value)); + } + return result; + } + + /** + * Write a value with its weighting, for example "gb (weighting 1.00)". + * Every weighting the examples print is written this way. + * @param value the value, may be null + * @param weighting the weighting, between 0 and 1 + * @return the value followed by its weighting + */ + public static String asWeightedString(Object value, float weighting) { + return String.format( + Locale.ROOT, + "%s (weighting %.2f)", + toDisplayString(value), + weighting); + } + + /** + * Write each value of an IP Intelligence property with its weighting, + * for example "gb (weighting 1.00)", or why there is no value. + * @param property the property value, may be null + * @return the values with their weightings, joined with ", " + */ + public static String asWeightedString(AspectPropertyValue property) { + if (property == null || property.hasValue() == false) { + return reportNoValue(property); + } + return weightedValues(property).stream() + .map(weighted -> asWeightedString( + weighted.getValue(), + weighted.getWeighting())) + .collect(Collectors.joining(", ")); + } + /** * Helper to get the first raw value of an IP Intelligence property. * Due to Java generics type erasure, the on-premise engine may return diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/settings.xml b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/settings.xml index 6cb8219ee..084f069ce 100644 --- a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/settings.xml +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/settings.xml @@ -45,11 +45,11 @@ your IDE launch configuration or your operating system shell. - + created at https://51degrees.com/configure/hYzn3TV3?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-ipintelligence-examples-shared-settings.xml&utm_term=properties. --> + <_51DEGREES_RESOURCE_KEY> diff --git a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/DataFileHelperTest.java b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/DataFileHelperTest.java index e1a3418b3..65993b935 100644 --- a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/DataFileHelperTest.java +++ b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/DataFileHelperTest.java @@ -68,43 +68,19 @@ public void findPerformanceProfile_Unknown_WritesNamesAccepted() { * variable. */ @Test - public void getDataFileLocation_OldNameWhenNewIsNotSet() - throws Exception { - assumePathNamesNotInEnvironment(); - String savedNew = System.clearProperty(NEW_PATH_NAME); - String savedOld = System.clearProperty(OLD_PATH_NAME); - File file = File.createTempFile("old-path-name", ".ipi"); - try { - System.setProperty(OLD_PATH_NAME, file.getPath()); - assertEquals(file.getPath(), - DataFileHelper.getDataFileLocation(null)); - } finally { - restore(NEW_PATH_NAME, savedNew); - restore(OLD_PATH_NAME, savedOld); - file.delete(); - } - } - - @Test - public void getDataFileLocation_NewNameWinsOverOld() throws Exception { - assumePathNamesNotInEnvironment(); + public void getDataFileLocation_NewName() throws Exception { + assumeTrue(System.getenv(NEW_PATH_NAME) == null); String savedNew = System.clearProperty(NEW_PATH_NAME); - String savedOld = System.clearProperty(OLD_PATH_NAME); File newFile = File.createTempFile("new-path-name", ".ipi"); - File oldFile = File.createTempFile("old-path-name", ".ipi"); try { System.setProperty(NEW_PATH_NAME, newFile.getPath()); - System.setProperty(OLD_PATH_NAME, oldFile.getPath()); assertEquals(newFile.getPath(), DataFileHelper.getDataFileLocation(null)); } finally { restore(NEW_PATH_NAME, savedNew); - restore(OLD_PATH_NAME, savedOld); newFile.delete(); - oldFile.delete(); } } - @Test @SuppressWarnings("deprecation") public void oldPathConstantKeepsItsValue() throws Exception { @@ -115,11 +91,6 @@ public void oldPathConstantKeepsItsValue() throws Exception { DataFileHelper.class.getField("IPI_PATH_VARIABLE").get(null)); } - private static void assumePathNamesNotInEnvironment() { - assumeTrue(System.getenv(NEW_PATH_NAME) == null); - assumeTrue(System.getenv(OLD_PATH_NAME) == null); - } - private static void restore(String name, String value) { if (value == null) { System.clearProperty(name); diff --git a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/FindOutMoreTest.java b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/FindOutMoreTest.java new file mode 100644 index 000000000..bb1d7843f --- /dev/null +++ b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/FindOutMoreTest.java @@ -0,0 +1,149 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.shared; + +import org.junit.Test; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class FindOutMoreTest { + + /** + * The campaign tags common-ci's scripts/utm-lint.ps1 requires of a link + * printed by an example. + */ + private static final Pattern TAGGED = Pattern.compile( + "https://51degrees\\.com/[^?\\s]+\\?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=[a-z0-9][a-z0-9._-]*" + + "&utm_term=[a-z0-9][a-z0-9._-]*"); + + private static String[] write(String example, String... repositories) { + StringWriter text = new StringWriter(); + FindOutMore.write(new PrintWriter(text), example, repositories); + return text.toString().split("\r?\n"); + } + + /** + * The section is a blank line, a rule, the heading, the three tagged + * pages, the repositories and a closing rule. + */ + @Test + public void write_LinksThenRepositories() { + String[] lines = write("console-example.java", + FindOutMore.ON_PREMISE_REPOSITORIES); + assertEquals(10, lines.length); + assertEquals("", lines[0]); + assertEquals(FindOutMore.RULE, lines[1]); + assertEquals(FindOutMore.HEADING, lines[2]); + assertTrue(lines[3].startsWith("\tIP Intelligence: https://51degrees.com/ip-intelligence?")); + assertTrue(lines[4].startsWith("\tPricing: https://51degrees.com/pricing?")); + assertTrue(lines[5].startsWith("\tDocumentation: https://51degrees.com/documentation/index.html?")); + for (int i = 3; i <= 5; i++) { + String link = lines[i].substring(lines[i].indexOf("https://")); + assertTrue(link, TAGGED.matcher(link).matches()); + assertTrue(link, link.contains("&utm_content=console-example.java&")); + } + assertEquals("\thttps://github.com/51Degrees/51degrees-java", lines[6]); + assertEquals("\thttps://github.com/51Degrees/51degrees-cxx", lines[7]); + assertEquals("\thttps://github.com/51Degrees/data", lines[8]); + assertEquals(FindOutMore.RULE, lines[9]); + } + + /** + * A cloud example lists this repository alone. + */ + @Test + public void write_Cloud_ThisRepository() { + String[] lines = write("console-example.java", + FindOutMore.CLOUD_REPOSITORIES); + assertEquals(8, lines.length); + assertEquals("\thttps://github.com/51Degrees/51degrees-java", lines[6]); + } + + /** + * Each line of the commented form starts with the comment marker, and + * there is no blank line, so a YAML stream stays valid. + */ + @Test + public void writeCommented_EveryLineAComment() { + StringWriter text = new StringWriter(); + FindOutMore.writeCommented(new PrintWriter(text), "# ", + "console-example.java", FindOutMore.CLOUD_REPOSITORIES); + String[] lines = text.toString().split("\\r?\\n"); + assertEquals(7, lines.length); + for (String line : lines) { + assertTrue(line, line.startsWith("# ")); + } + assertEquals("# " + FindOutMore.HEADING, lines[1]); + } + + /** + * The HTML form has the heading, each tagged page as a link with its + * ampersands escaped, and each repository as a link. + */ + @Test + public void html_LinksThenRepositories() { + String html = FindOutMore.html("web-example.java", + FindOutMore.ON_PREMISE_REPOSITORIES); + assertTrue(html, html.contains("

    " + FindOutMore.HEADING + "

    ")); + java.util.regex.Matcher href = + Pattern.compile("href=\"([^\"]+)\"").matcher(html); + int pages = 0; + int repositories = 0; + while (href.find()) { + String link = href.group(1); + if (link.startsWith("https://51degrees.com/")) { + assertTrue(link, link.contains("&")); + assertTrue(link, + TAGGED.matcher(link.replace("&", "&")).matches()); + pages++; + } else { + assertTrue(link, link.startsWith("https://github.com/51Degrees/")); + repositories++; + } + } + assertEquals(3, pages); + assertEquals(3, repositories); + } + + /** + * A tag that is not a lower case slug would fail the campaign tag + * convention, so it is refused. + */ + @Test + public void link_NotASlug_Refused() { + assertThrows(IllegalArgumentException.class, + () -> FindOutMore.link("pricing", "Console-Example.java")); + assertThrows(IllegalArgumentException.class, + () -> FindOutMore.link("pricing", "console example")); + assertThrows(IllegalArgumentException.class, + () -> FindOutMore.link("pricing", null)); + } +} diff --git a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java index df25ee4b2..c45e549ba 100644 --- a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java +++ b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java @@ -22,15 +22,19 @@ package fiftyone.ipintelligence.examples.shared; +import org.junit.AssumptionViolatedException; import org.junit.Test; +import java.net.URL; +import java.net.URLClassLoader; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assume.assumeTrue; @@ -42,12 +46,14 @@ public class KeyHelperTest { private static final String KEY_A = "AQAAAAAAAAAAAAAAAAAAAAAA"; private static final String KEY_B = "AgAAAAAAAAAAAAAAAAAAAAAA"; + private static final String NEW_KEY_NAME = "_51DEGREES_RESOURCE_KEY"; + private static final String OLD_KEY_NAME = "51DEGREES_RESOURCE_KEY"; + /** - * The names getOrSetTestResourceKey reads, in its order, apart from - * the deprecated name it reads when the first is not set. + * The names getOrSetTestResourceKey reads, in its order. */ - private static final List EXAMPLE_NAMES = Arrays.asList( - KeyHelper.RESOURCE_KEY_VARIABLE, KeyHelper.TEST_RESOURCE_KEY); + private static final List EXAMPLE_NAMES = + Collections.singletonList(KeyHelper.RESOURCE_KEY_VARIABLE); /** * With no key anywhere, the lookup logs that there is none and stores @@ -55,10 +61,7 @@ public class KeyHelperTest { */ @Test public void getOrSetTestResourceKey_NoKey_StoresNothing() { - // The deprecated name is read as well, so it must not be set - // either. List namesRead = new ArrayList<>(EXAMPLE_NAMES); - namesRead.add(OLD_KEY_NAME); // A test can clear a system property but not an environment // variable. for (String name : namesRead) { @@ -85,75 +88,73 @@ public void getOrSetTestResourceKey_NoKey_StoresNothing() { } @Test - public void requireResourceKey_NoKey_FailsNamingEveryName() { - AssertionError error = assertThrows(AssertionError.class, () -> - KeyHelper.requireResourceKey(name -> null, EXAMPLE_NAMES, - "the network and location of an IP address, and whether a " + - "device is mobile", - "ip", "device")); + public void testResourceKey_IsTheSharedName() { + assertEquals(NEW_KEY_NAME, KeyHelper.TEST_RESOURCE_KEY); + } + + /** + * A test with no key is skipped, naming the variable to set, rather + * than failed. + */ + @Test + public void requireResourceKey_NoKey_SkipsNamingTheVariable() { + AssumptionViolatedException skipped = assertThrows( + AssumptionViolatedException.class, () -> + KeyHelper.requireResourceKey(name -> null, EXAMPLE_NAMES, + "the network and location of an IP address, and " + + "whether a device is mobile", + "ip", "device")); assertEquals("This test calls the live 51Degrees cloud service, so " + - "it needs a resource key and there is nothing to fall back " + - "on. None of _51DEGREES_RESOURCE_KEY, " + - "_51DEGREES_RESOURCE_KEY_BESPOKE, _51DEGREES_RESOURCE_KEY, " + - "TestResourceKey is set, as an environment variable or as a " + - "system property, tried in that order. The key has to carry " + - "the ip and device products, because the test asks the " + - "service for the network and location of an IP address, and " + - "whether a device is mobile. To see the products a key " + - "carries, open " + + "it needs a resource key, and it is skipped because " + + "_51DEGREES_RESOURCE_KEY is not set as an environment " + + "variable or as a system property. The key has to carry the " + + "ip and device products, because the test asks the service " + + "for the network and location of an IP address, and whether " + + "a device is mobile. To see the products a key carries, open " + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=. " + "Create a key at " + KeyHelper.CREATE_KEY_URL + ".", - error.getMessage()); + skipped.getMessage()); } /** - * A name holding something that is not a key, such as the empty - * string the example poms pass for a key that is not set, is passed - * over. + * Only the names the test gives are read, and a name holding something + * that is not a key, such as the empty string a pom passes for a key + * that is not set, is passed over. */ @Test - public void requireResourceKey_FirstValidNameWins() { + public void requireResourceKey_ReadsOnlyTheNamesGiven() { Map values = new HashMap<>(); - values.put("_51DEGREES_RESOURCE_KEY", ""); + values.put(NEW_KEY_NAME, ""); values.put("_51DEGREES_RESOURCE_KEY_BESPOKE", KEY_A); - values.put(KeyHelper.TEST_RESOURCE_KEY, KEY_B); - assertEquals(KEY_A, KeyHelper.requireResourceKey(values::get, - EXAMPLE_NAMES, "every IP Intelligence property", "ip")); - } - - @Test - public void requireResourceKey_ExampleNamesInTheirOrder() { - Map values = new HashMap<>(); - values.put(KeyHelper.RESOURCE_KEY_VARIABLE, KEY_A); - values.put(KeyHelper.TEST_RESOURCE_KEY, KEY_B); - assertEquals(KEY_A, KeyHelper.requireResourceKey(values::get, - EXAMPLE_NAMES, "every IP Intelligence property", "ip")); - values.remove(KeyHelper.RESOURCE_KEY_VARIABLE); + assertThrows(AssumptionViolatedException.class, () -> + KeyHelper.requireResourceKey(values::get, EXAMPLE_NAMES, + "every IP Intelligence property", "ip")); + values.put(NEW_KEY_NAME, KEY_B); assertEquals(KEY_B, KeyHelper.requireResourceKey(values::get, EXAMPLE_NAMES, "every IP Intelligence property", "ip")); } - private static final String NEW_KEY_NAME = "_51DEGREES_RESOURCE_KEY"; - private static final String OLD_KEY_NAME = "51DEGREES_RESOURCE_KEY"; - - @Test - public void requireResourceKey_OldNameWhenNewIsNotSet() { - Map values = new HashMap<>(); - values.put(OLD_KEY_NAME, KEY_A); - values.put(KeyHelper.TEST_RESOURCE_KEY, KEY_B); - assertEquals(KEY_A, KeyHelper.requireResourceKey(values::get, - Arrays.asList(NEW_KEY_NAME, KeyHelper.TEST_RESOURCE_KEY), - "every IP Intelligence property", "ip")); - } - + /** + * A cloud example reads its key from _51DEGREES_RESOURCE_KEY alone. + */ @Test - public void requireResourceKey_NewNameWinsOverOld() { - Map values = new HashMap<>(); - values.put(NEW_KEY_NAME, KEY_B); - values.put(OLD_KEY_NAME, KEY_A); - assertEquals(KEY_B, KeyHelper.requireResourceKey(values::get, - Arrays.asList(NEW_KEY_NAME, KeyHelper.TEST_RESOURCE_KEY), - "every IP Intelligence property", "ip")); + public void findResourceKey_ReadsOnlyTheSharedName() { + assumeTrue(System.getenv(NEW_KEY_NAME) == null); + String other = "AnyOtherResourceKeyName"; + String saved = System.clearProperty(NEW_KEY_NAME); + try { + System.setProperty(other, KEY_A); + assertNull(KeyHelper.findResourceKey(null)); + System.setProperty(NEW_KEY_NAME, KEY_B); + assertEquals(KEY_B, KeyHelper.findResourceKey(null)); + } finally { + System.clearProperty(other); + if (saved == null) { + System.clearProperty(NEW_KEY_NAME); + } else { + System.setProperty(NEW_KEY_NAME, saved); + } + } } @Test @@ -165,4 +166,45 @@ public void oldKeyConstantKeepsItsValue() throws Exception { assertEquals(NEW_KEY_NAME, KeyHelper.class.getField("RESOURCE_KEY_VARIABLE").get(null)); } + + /** + * A cloud example runs as an application, with no test library on its + * class path, and reads its key with KeyHelper. So KeyHelper has to + * load, and find a key, where JUnit cannot be found. + */ + @Test + public void findResourceKey_NoJUnitOnTheClassPath_FindsTheKey() + throws Exception { + try (URLClassLoader loader = withoutJUnit()) { + Class helper = Class.forName(KeyHelper.class.getName(), true, + loader); + assertNotSame(KeyHelper.class, helper); + String key = (String) helper + .getMethod("findResourceKey", String.class) + .invoke(null, KEY_A); + assertEquals(KEY_A, key); + } + } + + /** + * A class loader with KeyHelper's own class files, that cannot see + * JUnit. + */ + private static URLClassLoader withoutJUnit() { + String helpers = KeyHelper.class.getPackage().getName() + "."; + ClassLoader parent = new ClassLoader( + KeyHelper.class.getClassLoader()) { + @Override + protected Class loadClass(String name, boolean resolve) + throws ClassNotFoundException { + if (name.startsWith("org.junit.") || + name.startsWith(helpers)) { + throw new ClassNotFoundException(name); + } + return super.loadClass(name, resolve); + } + }; + return new URLClassLoader(new URL[]{KeyHelper.class + .getProtectionDomain().getCodeSource().getLocation()}, parent); + } } diff --git a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/PropertyHelperMissingPropertyTests.java b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/PropertyHelperMissingPropertyTests.java new file mode 100644 index 000000000..5ebdd36b2 --- /dev/null +++ b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/PropertyHelperMissingPropertyTests.java @@ -0,0 +1,80 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.shared; + +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyReason; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * A Property the data file or resource key in use cannot give is answered + * with plain text that names it and gives the tagged contact link, because + * the examples print the answer to a console, where HTML is read as it is + * written. + */ +public class PropertyHelperMissingPropertyTests { + + private static final String CONTACT_LINK = + "https://51degrees.com/contact-us?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=shared-src-main-java-fiftyone-ipintelligence-examples-shared-propertyhelper.java"; + + private static AspectPropertyValue missing() { + return PropertyHelper.tryGet(() -> { + throw new PropertyMissingException( + MissingPropertyReason.DataFileUpgradeRequired, + "HardwareVendor", + "HardwareVendor is not in this data file."); + }); + } + + @Test + public void MissingProperty_IsPlainText() { + String message = missing().getNoValueMessage(); + assertFalse("The message holds HTML: " + message, + message.contains("<") || message.contains("&")); + } + + @Test + public void MissingProperty_NamesItAndGivesTheContactLink() { + String message = missing().getNoValueMessage(); + assertTrue("The message does not name the property: " + message, + message.contains("'HardwareVendor'")); + assertTrue("The message has no tagged contact link: " + message, + message.contains(CONTACT_LINK)); + } + + @Test + public void PricingMessage_GivesTheContactLink() { + assertTrue("The advisory has no tagged contact link: " + + PropertyHelper.PRICING_MESSAGE, + PropertyHelper.PRICING_MESSAGE.contains(CONTACT_LINK)); + assertFalse("The advisory holds HTML: " + + PropertyHelper.PRICING_MESSAGE, + PropertyHelper.PRICING_MESSAGE.contains("<")); + } +} diff --git a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/PropertyHelperTest.java b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/PropertyHelperTest.java index 705ceff72..c1bcc0d3d 100644 --- a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/PropertyHelperTest.java +++ b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/PropertyHelperTest.java @@ -22,11 +22,16 @@ package fiftyone.ipintelligence.examples.shared; +import fiftyone.pipeline.core.data.IWeightedValue; +import fiftyone.pipeline.core.data.WeightedValue; import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; import org.junit.Test; +import java.net.InetAddress; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import static org.junit.Assert.*; @@ -60,4 +65,42 @@ public void testAsNoValue() { AspectPropertyValue> test = new AspectPropertyValueDefault<>(); assertTrue(PropertyHelper.asString(test).startsWith("Unknown")); } + + @Test + public void testAsIPAddressProperty() throws Exception { + AspectPropertyValue test = new AspectPropertyValueDefault<>( + InetAddress.getByAddress(new byte[]{8, 8, 8, 0})); + assertEquals("8.8.8.0", PropertyHelper.asIPAddressProperty(test)); + } + + @Test + public void testAsStringWeightedIPAddress() throws Exception { + List> addresses = new ArrayList<>(); + addresses.add(new WeightedValue<>(65535, + InetAddress.getByAddress(new byte[]{8, 8, 8, (byte) 255}))); + AspectPropertyValue>> test = + new AspectPropertyValueDefault<>(addresses); + assertEquals("8.8.8.255 (weighting 1.00)", PropertyHelper.asString(test)); + } + + /** + * Each weighted value is shown with its weighting, as the IP + * Intelligence examples must show both. A raw weighting of 65535 is a + * weighting of 1. + */ + @Test + public void testWeightedValuesShowTheirWeighting() { + AspectPropertyValue>> one = + new AspectPropertyValueDefault<>(Collections.singletonList( + new WeightedValue<>(65535, "Japan"))); + assertEquals("Japan (weighting 1.00)", PropertyHelper.asString(one)); + + AspectPropertyValue>> two = + new AspectPropertyValueDefault<>(Arrays.asList( + new WeightedValue<>(49151, "Tokyo"), + new WeightedValue<>(16384, "Osaka"))); + assertEquals( + "Tokyo (weighting 0.75), Osaka (weighting 0.25)", + PropertyHelper.asString(two)); + } } \ No newline at end of file diff --git a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/PropertyHelperWeightingTests.java b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/PropertyHelperWeightingTests.java new file mode 100644 index 000000000..b8f93d5d6 --- /dev/null +++ b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/PropertyHelperWeightingTests.java @@ -0,0 +1,111 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.shared; + +import fiftyone.pipeline.core.data.IWeightedValue; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * "Examples MUST demonstrate accessing both the value and its associated + * weighting" (ip-intelligence-specification/required-examples.md line 107). + * The examples read each value of a property with its weighting through + * {@link PropertyHelper#weightedValues}. + */ +public class PropertyHelperWeightingTests { + + private static IWeightedValue weighted( + final int rawWeighting, + final Object value) { + return new IWeightedValue() { + @Override + public int getRawWeighting() { + return rawWeighting; + } + + @Override + public Object getValue() { + return value; + } + }; + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + @Test + public void weightedValues_ListOfWeightedValues_KeepsEachWeighting() { + List> list = new ArrayList<>(); + list.add(weighted(49151, "gb")); + list.add(weighted(16384, "ie")); + AspectPropertyValue property = + new AspectPropertyValueDefault(list); + + List> values = + PropertyHelper.weightedValues(property); + + assertEquals(2, values.size()); + assertEquals("gb", values.get(0).getValue()); + assertEquals(49151 / 65535f, values.get(0).getWeighting(), 0.0001); + assertEquals("ie", values.get(1).getValue()); + assertEquals(16384 / 65535f, values.get(1).getWeighting(), 0.0001); + } + + /** + * "a property with a single answer returns it with a weighting of 1" + * (required-examples.md line 107) + */ + @Test + public void weightedValues_SingleValue_HasWeightingOfOne() { + AspectPropertyValue property = + new AspectPropertyValueDefault<>("gb"); + + List> values = + PropertyHelper.weightedValues(property); + + assertEquals(1, values.size()); + assertEquals("gb", values.get(0).getValue()); + assertEquals(1f, values.get(0).getWeighting(), 0.0001); + } + + @Test + public void weightedValues_NoValue_IsEmpty() { + assertTrue(PropertyHelper.weightedValues( + new AspectPropertyValueDefault()).isEmpty()); + assertTrue(PropertyHelper.weightedValues(null).isEmpty()); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + @Test + public void asWeightedString_ShowsValueAndWeighting() { + List> list = new ArrayList<>(); + list.add(weighted(65535, "gb")); + assertEquals("gb (weighting 1.00)", PropertyHelper.asWeightedString( + new AspectPropertyValueDefault(list))); + } +} diff --git a/examples/ip-intelligence/shared/src/test/resources/51Degrees-Cloud-Test.xml b/examples/ip-intelligence/shared/src/test/resources/51Degrees-Cloud-Test.xml index 3af0fba3c..4e16e3e9e 100644 --- a/examples/ip-intelligence/shared/src/test/resources/51Degrees-Cloud-Test.xml +++ b/examples/ip-intelligence/shared/src/test/resources/51Degrees-Cloud-Test.xml @@ -27,7 +27,7 @@ http://cloud.51degrees.com/api/v4 + + + dev + + ${project.version} + + + + diff --git a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java index 34b00b036..1d36cd8d1 100644 --- a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java +++ b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java @@ -50,6 +50,7 @@ import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngineBuilder; import fiftyone.ipintelligence.examples.shared.DataFileHelper; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.FlowElement; @@ -74,6 +75,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.PrintStream; +import java.io.PrintWriter; import java.nio.file.Files; import java.nio.file.Paths; import java.security.MessageDigest; @@ -94,17 +97,31 @@ * and a JSON builder element which serialises the results. *

    * The data file can be supplied as a command line argument. When no argument is given - * the enterprise data file is looked for in the cxx/data/ip-intelligence directory. + * the free Lite data file is used, and downloaded when it is missing. + *

    + * The service answers in JSON, so when it starts it writes what the Enterprise data file + * adds, when the file in use is a free one, and the "Find out more" section. *

    * Note that the .NET version of this example also includes a device detection engine * in the same pipeline. This Java version is IP Intelligence only. */ public class GettingStartedApi extends HttpServlet { private static final long serialVersionUID = 1734154705981153543L; + // A local stand-in for the cloud service, not a web page, so it keeps + // 5225, as the .NET version does: a cloud example pointed at it can run + // beside a web example on 5151. public static final int DEFAULT_PORT = 5225; public static final String DATA_FILE_INIT_PARAM = "dataFile"; public static Logger logger = LoggerFactory.getLogger(GettingStartedApi.class); + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "web-getting-started.api-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedapi.java"; + + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=web-getting-started.api-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedapi.java&utm_term=enterprise-data-file"; + private Pipeline pipeline; private IPIntelligenceOnPremiseEngine engine; @@ -153,6 +170,25 @@ public void init() throws ServletException { throw new ServletException("Failed to create pipeline from data file '" + dataFileLocation + "'", e); } + writeStartupNotes(System.out, engine); + } + + /** + * Writes what the Enterprise data file adds, with the link to the + * contact page, when the engine uses a free data file, then the "Find + * out more" section. + * @param output where to write + * @param engine the engine the service uses + */ + static void writeStartupNotes( + PrintStream output, + IPIntelligenceOnPremiseEngine engine) { + if (DataFileHelper.isFreeDataFile(engine)) { + DataFileHelper.writeEnterpriseOffer( + new PrintWriter(output), CONTACT_US); + } + FindOutMore.write(output, EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES); } @Override @@ -228,6 +264,9 @@ private void processEvidence(HttpServletRequest request, HttpServletResponse res } } + // A Flow Data cannot be used by two threads at once. Work handed to + // another thread or task gets the values it needs, read here on the + // request's thread, not the Flow Data. try (FlowData flowData = pipeline.createFlowData()) { flowData.addEvidence(evidence); flowData.process(); diff --git a/examples/ip-intelligence/web/getting-started.api/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedApiNotesTest.java b/examples/ip-intelligence/web/getting-started.api/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedApiNotesTest.java new file mode 100644 index 000000000..897d74f93 --- /dev/null +++ b/examples/ip-intelligence/web/getting-started.api/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedApiNotesTest.java @@ -0,0 +1,80 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.web; + +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngineBuilder; +import fiftyone.ipintelligence.examples.shared.DataFileHelper; +import fiftyone.ipintelligence.examples.shared.FindOutMore; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import static org.junit.Assert.assertTrue; + +/** + * The service answers in JSON, so what it writes when it starts is the only + * place the person running it reads what the Enterprise data file adds and + * where to find out more. + */ +public class GettingStartedApiNotesTest { + + private static final String CONTACT_US = "https://51degrees.com/contact-us?"; + + /** + * On the free Lite data file, the service says what the Enterprise data + * file adds with the link to the contact page, then writes the "Find out + * more" section, closed by its rule. + */ + @Test + public void startupNotes_FreeDataFile() throws Exception { + String lite = DataFileHelper.getDataFileLocation( + DataFileHelper.LITE_DATA_FILE_REL_PATH); + try (IPIntelligenceOnPremiseEngine engine = + new IPIntelligenceOnPremiseEngineBuilder( + LoggerFactory.getILoggerFactory()) + .setAutoUpdate(false) + .build(lite, false)) { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + GettingStartedApi.writeStartupNotes( + new PrintStream(buffer, true, "UTF-8"), engine); + String output = buffer.toString("UTF-8"); + + assertTrue("The output should say what the Enterprise data " + + "file adds:\n" + output, + output.contains("This example used a free data file.")); + assertTrue("The output should link the contact page:\n" + output, + output.contains(CONTACT_US)); + int heading = output.lastIndexOf(FindOutMore.HEADING); + assertTrue("The output should have the '" + FindOutMore.HEADING + + "' section after the link to the contact " + + "page:\n" + output, + heading > output.lastIndexOf(CONTACT_US)); + assertTrue("The '" + FindOutMore.HEADING + + "' section should end with its rule:\n" + output, + output.indexOf(FindOutMore.RULE, heading) > heading); + } + } +} diff --git a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java index 977e3b3fa..76daccd62 100644 --- a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java +++ b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixed.java @@ -47,6 +47,7 @@ package fiftyone.ipintelligence.examples.web; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.devicedetection.shared.DeviceData; import fiftyone.ipintelligence.examples.shared.KeyHelper; import fiftyone.ipintelligence.shared.IPIntelligenceData; @@ -83,11 +84,23 @@ *

    * The configuration file for the pipeline is at * src/main/webapp/WEB-INF/51Degrees-CloudMixed.xml. The resource key is taken from the - * environment variable or system property "TestResourceKey" unless supplied as a command - * line argument. + * environment variable or system property "_51DEGREES_RESOURCE_KEY" unless supplied as a command + * line argument. The server listens on the port in the PORT environment variable, or on + * 5151 when PORT is not set. */ public class GettingStartedWebCloudMixed extends HttpServlet { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "web-getting-started.cloud.mixed-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedwebcloudmixed.java"; + private static final long serialVersionUID = 1734154705981153543L; + /** + * The port the example listens on when the PORT environment variable + * does not choose another. + */ + public static final int DEFAULT_PORT = 5151; public static Logger logger = LoggerFactory.getLogger(GettingStartedWebCloudMixed.class); public static void main(String[] args) throws Exception { @@ -95,12 +108,27 @@ public static void main(String[] args) throws Exception { logger.info("Running Example {}", GettingStartedWebCloudMixed.class); // Get the resource key from the command line, environment variable or - // system property. This sets the "TestResourceKey" system property + // system property. This sets the "_51DEGREES_RESOURCE_KEY" system property // which is interpolated in the pipeline configuration file. - KeyHelper.getOrSetTestResourceKey(args.length > 0 ? args[0] : null, true); + String resourceKey = KeyHelper.findResourceKey( + args.length > 0 ? args[0] : null); + if (resourceKey == null) { + KeyHelper.writeNoResourceKey(System.out, EXAMPLE_TAG); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); + return; + } + KeyHelper.getOrSetTestResourceKey(resourceKey, true); - // Start Jetty with this WebApp - EmbedJetty.runWebApp(getResourceBase(), 8084); + String portEnv = System.getenv("PORT"); + if (portEnv != null) { + // An automated run sets PORT and has no console input, so keep + // the server running by joining it rather than waiting on Enter. + EmbedJetty.startWebApp(getResourceBase(), Integer.parseInt(portEnv)).join(); + } else { + // Start Jetty with this WebApp + EmbedJetty.runWebApp(getResourceBase(), DEFAULT_PORT); + } } public static String getResourceBase() { @@ -121,6 +149,9 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re // The detection has already been carried out by the PipelineFilter, // which closes the FlowData at the end of the request, so it is not // closed here. + // A Flow Data cannot be used by two threads at once. Work handed to + // another thread or task gets the values it needs, read here on the + // request's thread, not the Flow Data. FlowData flowData = flowDataProvider.getFlowData(request); // Determine target IP for display. @@ -194,6 +225,8 @@ private String substituteTemplateValues(String template, DeviceData deviceData, .replace("${AREAS_JS}", escapeForJs(asWktStringProperty(tryGet(ipiData::getAreas)))) .replace("${ACCURACY_RADIUS}", asIntegerProperty(tryGet(ipiData::getAccuracyRadiusMin))) .replace("${TIME_ZONE_OFFSET}", asIntegerProperty(tryGet(ipiData::getTimeZoneOffset))) + .replace("${FIND_OUT_MORE}", FindOutMore.html(EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES)) .replace("${EVIDENCE_ROWS}", buildEvidenceRows(flowData)) .replace("${LocationConfidence}", asStringProperty(tryGet(ipiData::getLocationConfidence))) .replace("${ConnectionType}", asStringProperty(tryGet(ipiData::getConnectionType))) diff --git a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/webapp/WEB-INF/51Degrees-CloudMixed.xml b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/webapp/WEB-INF/51Degrees-CloudMixed.xml index 1c3244538..f5e4513c2 100644 --- a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/webapp/WEB-INF/51Degrees-CloudMixed.xml +++ b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/webapp/WEB-INF/51Degrees-CloudMixed.xml @@ -27,9 +27,9 @@ - ${TestResourceKey} + ${_51DEGREES_RESOURCE_KEY} CloudRequestEngine diff --git a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/webapp/WEB-INF/html/index.html b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/webapp/WEB-INF/html/index.html index 0c6a897fe..6e4bafcbd 100644 --- a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/webapp/WEB-INF/html/index.html +++ b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/webapp/WEB-INF/html/index.html @@ -4,6 +4,7 @@ Device Detection + IP Intelligence Example + @@ -100,10 +101,11 @@

    IP Intelligence Results


    -
    + The shared examples.js helper appends a results table to this + section after client-side evidence has been collected and + processed. This provides more accurate detection for properties + like screen size and Apple device models. --> +

    Client-Side Enhanced Detection

    Results will appear here after client-side evidence collection completes.

    @@ -133,6 +135,8 @@

    Evidence Used

    + ${FIND_OUT_MORE} + @@ -152,8 +156,8 @@

    Evidence Used

    if (isValidValue(areasWkt) && areasWkt !== EMPTY_POLYGON) { document.getElementById('map-section').style.display = 'block'; var map = L.map('map'); - L.tileLayer('https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png', { - attribution: '© OpenStreetMap contributors © CARTO', + L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', { + attribution: '© OpenStreetMap contributors', subdomains: 'abcd', maxZoom: 19 }).addTo(map); @@ -183,8 +187,8 @@

    Evidence Used

    var lng = parseFloat(longitude); if (!isNaN(lat) && !isNaN(lng)) { var map = L.map('map').setView([lat, lng], 10); - L.tileLayer('https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png', { - attribution: '© OpenStreetMap contributors © CARTO', + L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', { + attribution: '© OpenStreetMap contributors', subdomains: 'abcd', maxZoom: 19 }).addTo(map); @@ -200,61 +204,16 @@

    Evidence Used

    and served by the PipelineFilter. It collects client-side evidence (screen size, hardware capabilities, etc.) for enhanced device detection. --> + diff --git a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/webapp/css/examples-main.min.css b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/webapp/css/examples-main.min.css new file mode 100644 index 000000000..42a9f40d7 --- /dev/null +++ b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/webapp/css/examples-main.min.css @@ -0,0 +1 @@ +/*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */html{line-height:1.15;-webkit-text-size-adjust:100%}body{margin:0}main{display:block}h1{font-size:2em;margin:.67em 0}hr{box-sizing:content-box;height:0;overflow:visible}pre{font-family:monospace,monospace;font-size:1em}a{background-color:transparent}abbr[title]{border-bottom:none;text-decoration:underline;text-decoration:underline dotted}b,strong{font-weight:bolder}code,kbd,samp{font-family:monospace,monospace;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}img{border-style:none}button,input,optgroup,select,textarea{font-family:inherit;font-size:100%;line-height:1.15;margin:0}button,input{overflow:visible}button,select{text-transform:none}button,[type=button],[type=reset],[type=submit]{-webkit-appearance:button}button::-moz-focus-inner,[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner{border-style:none;padding:0}button:-moz-focusring,[type=button]:-moz-focusring,[type=reset]:-moz-focusring,[type=submit]:-moz-focusring{outline:1px dotted ButtonText}fieldset{padding:.35em .75em .625em}legend{box-sizing:border-box;color:inherit;display:table;max-width:100%;padding:0;white-space:normal}progress{vertical-align:baseline}textarea{overflow:auto}[type=checkbox],[type=radio]{box-sizing:border-box;padding:0}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}details{display:block}summary{display:list-item}template{display:none}[hidden]{display:none}.a-size--10{width:.1615055829rem;height:.1615055829rem}.a-size--9{width:.1938066995rem;height:.1938066995rem}.a-size--8{width:.2325680394rem;height:.2325680394rem}.a-size--7{width:.2790816472rem;height:.2790816472rem}.a-size--6{width:.3348979767rem;height:.3348979767rem}.a-size--5{width:.401877572rem;height:.401877572rem}.a-size--4{width:.4822530864rem;height:.4822530864rem}.a-size--3{width:.5787037037rem;height:.5787037037rem}.a-size--2{width:.6944444444rem;height:.6944444444rem}.a-size--1{width:.8333333333rem;height:.8333333333rem}.a-size-0{width:1rem;height:1rem}.a-size-1{width:1.2rem;height:1.2rem}.a-size-2{width:1.44rem;height:1.44rem}.a-size-3{width:1.728rem;height:1.728rem}.a-size-4{width:2.0736rem;height:2.0736rem}.a-size-5{width:2.48832rem;height:2.48832rem}.a-size-6{width:2.985984rem;height:2.985984rem}.a-size-7{width:3.5831808rem;height:3.5831808rem}.a-size-8{width:4.29981696rem;height:4.29981696rem}.a-size-9{width:5.159780352rem;height:5.159780352rem}.a-size-10{width:6.1917364224rem;height:6.1917364224rem}.a-size-11{width:7.4300837069rem;height:7.4300837069rem}.a-size-12{width:8.9161004483rem;height:8.9161004483rem}.a-size-13{width:10.6993205379rem;height:10.6993205379rem}.a-size-14{width:12.8391846455rem;height:12.8391846455rem}.a-size-15{width:15.4070215746rem;height:15.4070215746rem}.a-size-16{width:18.4884258895rem;height:18.4884258895rem}.a-size-17{width:22.1861110674rem;height:22.1861110674rem}.a-size-18{width:26.6233332809rem;height:26.6233332809rem}.a-size-19{width:31.9479999371rem;height:31.9479999371rem}.a-size-20{width:38.3375999245rem;height:38.3375999245rem}.a-font-size--10{font-size:.1615055829rem}.a-font-size--9{font-size:.1938066995rem}.a-font-size--8{font-size:.2325680394rem}.a-font-size--7{font-size:.2790816472rem}.a-font-size--6{font-size:.3348979767rem}.a-font-size--5{font-size:.401877572rem}.a-font-size--4{font-size:.4822530864rem}.a-font-size--3{font-size:.5787037037rem}.a-font-size--2{font-size:.6944444444rem}.a-font-size--1{font-size:.8333333333rem}.a-font-size-0{font-size:1rem}.a-font-size-1{font-size:1.2rem}.a-font-size-2{font-size:1.44rem}.a-font-size-3{font-size:1.728rem}.a-font-size-4{font-size:2.0736rem}.a-font-size-5{font-size:2.48832rem}.a-font-size-6{font-size:2.985984rem}.a-font-size-7{font-size:3.5831808rem}.a-font-size-8{font-size:4.29981696rem}.a-font-size-9{font-size:5.159780352rem}.a-font-size-10{font-size:6.1917364224rem}.a-font-size-11{font-size:7.4300837069rem}.a-font-size-12{font-size:8.9161004483rem}.a-font-size-13{font-size:10.6993205379rem}.a-font-size-14{font-size:12.8391846455rem}.a-font-size-15{font-size:15.4070215746rem}.a-font-size-16{font-size:18.4884258895rem}.a-font-size-17{font-size:22.1861110674rem}.a-font-size-18{font-size:26.6233332809rem}.a-font-size-19{font-size:31.9479999371rem}.a-font-size-20{font-size:38.3375999245rem}.a-font--default{font-family:Arial,sans-serif}.a-font--code{font-family:Consolas,monospace}.b-text--hidden{position:absolute;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}.b-text--default{font-family:Arial,sans-serif;font-size:1rem;color:#252525}.b-text--count{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#9b9b9b}.b-text--count-pill{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#9b9b9b;border:1px solid #BAC82A;border-radius:28px;padding:.093463879rem .4822530864rem;color:#252525;background-color:#fff;display:inline}.b-text--property-pill{font-family:Arial,sans-serif;font-size:.8333333333rem;color:#646060}.b-text--heading-1{font-family:Arial,sans-serif;font-weight:700;font-size:1.44rem;color:#252525;margin:0 0 1.728rem}.b-text--heading-2{font-family:Arial,sans-serif;font-weight:700;font-size:1.2rem;color:#252525;margin:0 0 1.44rem}.b-text--heading-3{font-family:Arial,sans-serif;font-weight:700;font-size:1rem;color:#252525;margin:0 0 1.2rem}.b-text--code{font-family:Consolas,monospace;font-size:.8333333333rem;color:#646060}.b-text--code-snip{font-family:Consolas,monospace;font-size:.8333333333rem;color:#646060;background-color:#fff;border-radius:5px;border:1px solid #EAEAEA;padding:0 .2325680394rem}.b-text--heading-caps{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#252525;font-weight:400;text-transform:uppercase;letter-spacing:.5px}.b-text--copyright{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#252525}.b-text--progress{font-family:Arial,sans-serif;font-size:.8333333333rem}@media(min-width:769px){.b-text--progress{font-size:1rem}}.b-text--progress{color:#252525;letter-spacing:.7px;text-transform:uppercase;font-weight:700}.b-text--red{color:#cc2b27}body{font-family:Arial,sans-serif;font-size:1rem;color:#252525;margin:0;line-height:1.5}*{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;box-sizing:border-box}.b-link{transition:color .3s ease;color:#252525;text-decoration:underline}.b-link:active,.b-link:hover,.b-link:focus{color:#f5841f;text-decoration:underline}.b-link{display:inline-flex;align-items:center}.b-link img{margin:0 .5787037037rem}.b-link--property{font-family:Arial,sans-serif;font-size:.8333333333rem;color:#646060;color:#9b9b9b;border-bottom:2px dotted #EAEAEA;text-decoration:none}.b-link--property:active,.b-link--property:hover,.b-link--property:focus{color:#646060;border-bottom:2px dotted #646060;text-decoration:none}.b-link--dotted{color:#9b9b9b;border-bottom:2px dotted #EAEAEA;text-decoration:none}.b-link--dotted:active,.b-link--dotted:hover,.b-link--dotted:focus{color:#646060;border-bottom:2px dotted #646060;text-decoration:none}.b-link--unstyled{color:inherit;text-decoration:none}.b-link--unstyled:active,.b-link--unstyled:hover,.b-link--unstyled:focus{color:inherit;text-decoration:none}.b-link--docs{transition:color .3s ease;color:#00aeef;text-decoration:none;font-weight:700}.b-link--docs:active,.b-link--docs:hover,.b-link--docs:focus{text-decoration:underline}a.anchor{scroll-margin-top:128px}.b-btn{font-family:Arial,sans-serif;font-size:1rem;color:#252525;color:inherit;text-decoration:none}.b-btn:active,.b-btn:hover,.b-btn:focus{color:inherit;text-decoration:none}.b-btn{display:inline-flex;justify-content:center;align-items:center;padding:.8333333333rem 1.728rem;border:1px solid #BAC82A;background-color:#cbdb2a;cursor:pointer}.b-btn:active,.b-btn:hover,.b-btn:focus{background-color:#daed1a;border:1px solid #CBDB2A;text-decoration:none}.b-btn.b-btn--disabled,.b-btn:disabled{background-color:#f8f8f8;border:1px solid #EAEAEA;color:#646060;cursor:not-allowed;pointer-events:all!important}@media(min-width:769px){.b-btn--large{padding:1.2rem 2.48832rem}}.b-btn--secondary{background-color:#fff}.b-btn--secondary:active,.b-btn--secondary:hover,.b-btn--secondary:focus{background-color:#fff}.b-btn--block{width:100%}.b-btn__icon{margin-right:1rem}.b-btn__icon--after{margin-right:0;margin-left:1rem}input[type=text]{-webkit-appearance:none;-moz-appearance:none;appearance:none}.b-label{color:#646060;font-size:.8333333333rem;margin-bottom:.4822530864rem;width:100%}.b-input-group{display:flex;flex-direction:column}@media(min-width:769px){.b-input-group{flex-direction:row}}.b-input{font-family:Arial,sans-serif;font-size:1rem;color:#252525;border:solid 1px #646060;padding:.8333333333rem;line-height:1.2}.b-form-group--with-icon .b-input{padding-left:46px}.b-input::placeholder{color:#bfbfbf;opacity:1}.b-input,.b-input--block{width:100%}.b-form-group{display:flex;flex-direction:column}.b-form-group .b-btn{margin-top:1rem}@media(min-width:769px){.b-form-group .b-btn{margin-top:0;margin-left:.8333333333rem}}.b-form-group--with-icon{position:relative}.b-form-group--with-icon .b-icon{position:absolute;width:20px;height:20px;top:14px;left:10px}.c-eg-page{max-width:800px;margin:0 auto;padding:1.44rem 1.2rem}.c-eg-page__title{font-family:Arial,sans-serif;font-weight:700;font-size:1.2rem;color:#252525;margin:0 0 1.44rem}.c-eg-page__heading{font-family:Arial,sans-serif;font-weight:700;font-size:1rem;color:#252525;margin:1.728rem 0 1.2rem}.c-eg-page__lead{margin-bottom:1.44rem}.c-eg-section{margin-bottom:1.728rem}.c-eg-alert{font-family:Arial,sans-serif;font-size:1rem;color:#252525;border:1px solid #CC2B27;border-radius:5px;background-color:#fef1f9;padding:.8333333333rem 1rem;margin:1.2rem 0}.c-eg-alert a{color:#cc2b27;font-weight:700}.c-eg-table{font-family:Arial,sans-serif;font-size:1rem;color:#252525;width:100%;max-width:800px;font-size:.8333333333rem;background-color:#fff;border-collapse:collapse;margin-bottom:1.2rem}.c-eg-table__head .c-eg-table__cell{font-weight:700;border-bottom:3px solid #EAEAEA}.c-eg-table__cell{padding:.5787037037rem .6944444444rem;text-align:left;vertical-align:top;border-bottom:1px solid #EAEAEA}.c-eg-table__cell--key{font-weight:700;white-space:nowrap}.c-eg-table__cell ul{margin:0;padding-left:1rem}.c-eg-table__row--used{background-color:#f9fbe9}.c-eg-table__row--present,.c-eg-table__row--alt{background-color:#f8f8f8}.c-eg-legend{font-size:.6944444444rem;color:#646060;margin-bottom:.6944444444rem}.c-eg-legend__swatch{padding:0 .4822530864rem;border-radius:5px}.c-eg-legend__swatch--used{background-color:#f9fbe9}.c-eg-legend__swatch--present{background-color:#f8f8f8}.c-eg-form{margin-bottom:1.44rem}.c-eg-form__row{display:flex;flex-direction:column}@media(min-width:769px){.c-eg-form__row{flex-direction:row;align-items:flex-end}}.c-eg-form .b-btn{margin-top:.8333333333rem}@media(min-width:769px){.c-eg-form .b-btn{margin-top:0;margin-left:.8333333333rem}}.c-eg-map{margin:1.44rem 0}.c-eg-map__title{font-family:Arial,sans-serif;font-weight:700;font-size:1rem;color:#252525;margin:0 0 1.2rem}.c-eg-map__canvas{height:400px;width:100%;border:1px solid #EAEAEA;border-radius:5px}.c-eg-columns{display:grid;grid-template-columns:1fr;gap:1.44rem}@media(min-width:993px){.c-eg-columns{grid-template-columns:1fr 1fr}}.c-eg-details{margin:1.2rem 0 1.44rem;border:1px solid #EAEAEA;border-radius:5px;padding:0 1rem}.c-eg-details>summary{cursor:pointer;padding:.6944444444rem 0;font-weight:700}.c-eg-button-row{margin:1.2rem 0 1.44rem}.c-eg-message{display:flex;flex-direction:column;align-items:flex-start;gap:.8333333333rem;margin-top:2.0736rem;padding:1rem 1.2rem;background:#f8f8f8;border:1px solid #EAEAEA;border-radius:5px}@media(min-width:769px){.c-eg-message{flex-direction:row;align-items:center;justify-content:space-between;gap:1.2rem}}.c-eg-message__text{margin:0}.c-eg-message__cta{flex-shrink:0} diff --git a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/webapp/js/examples.min.js b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/webapp/js/examples.min.js new file mode 100644 index 000000000..db9929460 --- /dev/null +++ b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/webapp/js/examples.min.js @@ -0,0 +1 @@ +(function(d){"use strict";var f={};f.bindDeviceCallback=function(e){if(e=e||{},!(typeof d.fod>"u"||!d.fod.complete)){var a=e.targetId||"content",r=e.fields||C;d.fod.complete(function(i){_(a,r(i))})}};function C(e){var a=e&&e.device||{},r=typeof a.hardwarename>"u"?"Unknown":[].concat(a.hardwarename).join(", ");return[["Hardware Name:",r],["Platform:",m(a.platformname,a.platformversion)],["Browser:",m(a.browsername,a.browserversion)],["Screen width (pixels):",a.screenpixelswidth],["Screen height (pixels):",a.screenpixelsheight],["Device Id:",a.deviceid]]}function m(e,a){return[e,a].filter(function(r){return r!=null&&r!==""}).join(" ")}function _(e,a){var r=document.getElementById(e);if(r){var i=document.createElement("table");i.className="c-eg-table";var t=document.createElement("tr");t.className="c-eg-table__row c-eg-table__head",o(t,"th","Key",!1),o(t,"th","Value",!1),i.appendChild(t),a.forEach(function(l){var n=document.createElement("tr");n.className="c-eg-table__row c-eg-table__row--used",o(n,"td",l[0],!0),o(n,"td",l[1],!1),i.appendChild(n)}),r.appendChild(i)}}function o(e,a,r,i){var t=document.createElement(a);t.className="c-eg-table__cell"+(i?" c-eg-table__cell--key":"");var l=document.createTextNode(r??"");if(i){var n=document.createElement("strong");n.appendChild(l),l=n}t.appendChild(l),e.appendChild(t)}f.initLocationMap=function(e){e=e||{};var a=d.L;if(a){var r=e.sectionId||"map-section",i=e.canvasId||"map",t=e.areasWkt||"",l=e.latitude,n=e.longitude,v=e.labels||{ipLocation:"IP Location",lat:"Lat",lng:"Lng"},h=e.tileUrl||"https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png",w={attribution:"\xA9 OpenStreetMap contributors \xA9 CARTO",subdomains:"abcd",maxZoom:19};if(c(t)&&t!=="POLYGON EMPTY"&&d.wellknown){u(r,!0);var s=a.map(i);a.tileLayer(h,w).addTo(s);try{var g=d.wellknown.parse(t);if(g){var E=a.geoJSON(g,{style:{color:"#CC2B27",weight:2,opacity:.8,fillColor:"#CC2B27",fillOpacity:.2}}).addTo(s);s.fitBounds(E.getBounds()),p(a,s,l,n,v)}}catch(k){console.error("Error parsing polygon:",k),u(r,!1)}}else if(c(l)&&c(n)){var b=parseFloat(l),N=parseFloat(n);if(!isNaN(b)&&!isNaN(N)){u(r,!0);var y=a.map(i).setView([b,N],10);a.tileLayer(h,w).addTo(y),p(a,y,l,n,v)}}}};function p(e,a,r,i,t){if(!(!c(r)||!c(i))){var l=parseFloat(r),n=parseFloat(i);isNaN(l)||isNaN(n)||e.marker([l,n]).addTo(a).bindPopup(t.ipLocation+"
    "+t.lat+": "+l+"
    "+t.lng+": "+n).openPopup()}}function u(e,a){var r=document.getElementById(e);r&&(r.style.display=a?"block":"none")}function c(e){return e&&String(e).indexOf("Unknown")===-1&&e!=="0"&&e!==""}d.fodExamples=f})(window); diff --git a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/DefaultPortTest.java b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/DefaultPortTest.java new file mode 100644 index 000000000..173552398 --- /dev/null +++ b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/DefaultPortTest.java @@ -0,0 +1,39 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.web; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Checks that the example listens on port 5151 when the PORT environment + * variable does not choose another. It needs no resource key, so it runs + * where the page test is skipped. + */ +public class DefaultPortTest { + @Test + public void listensOn5151ByDefault() { + assertEquals(5151, GettingStartedWebCloudMixed.DEFAULT_PORT); + } +} diff --git a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedTest.java b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedTest.java index ff49ca9e7..a5b7af546 100644 --- a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedTest.java +++ b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedTest.java @@ -22,6 +22,7 @@ package fiftyone.ipintelligence.examples.web; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.ipintelligence.examples.shared.KeyHelper; import org.eclipse.jetty.server.Server; import org.junit.AfterClass; @@ -90,6 +91,9 @@ public void testWebCloudMixed() throws Exception { assertEquals("Expected HTTP 200 OK response", 200, code); assertTrue("Response should not be empty", responseBody.length() > 0); + assertTrue("The page should end with the '" + FindOutMore.HEADING + + "' section", responseBody.contains( + "

    " + FindOutMore.HEADING + "

    ")); } @AfterClass diff --git a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedWithoutKeyTest.java b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedWithoutKeyTest.java new file mode 100644 index 000000000..46b28eb04 --- /dev/null +++ b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedWithoutKeyTest.java @@ -0,0 +1,67 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.web; + +import fiftyone.ipintelligence.examples.shared.ExampleProcess; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Runs the IP Intelligence cloud and on-premise web example as it runs from + * a fresh clone, with no resource key anywhere, and checks that it ends + * without an error, names the variable to set and links to where a key is + * created, then ends with its Find out more section. + */ +public class GettingStartedWebCloudMixedWithoutKeyTest { + + private static final String CONFIGURE = "https://51degrees.com/configure/"; + + private static final String TAGS = + "?utm_source=code&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content="; + + private static void runWithoutKey(Class example) throws Exception { + ExampleProcess.Result result = ExampleProcess.runWithoutKeys(example); + System.out.println(result.output); + String output = result.output; + assertEquals(example.getSimpleName() + " should end with exit code 0", + 0, result.exitCode); + assertTrue("The output should name _51DEGREES_RESOURCE_KEY", + output.contains("Set the _51DEGREES_RESOURCE_KEY " + + "environment variable")); + int link = output.indexOf(CONFIGURE); + assertTrue("The output should link to where a key is created", + link >= 0 && output.startsWith(TAGS, + output.indexOf('?', link))); + assertTrue("The output should end with the Find out more section " + + "after the link", + output.lastIndexOf("Find out more") > link); + } + + @Test + public void gettingStartedWebCloudMixed() throws Exception { + runWithoutKey(GettingStartedWebCloudMixed.class); + } +} diff --git a/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java b/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java index 2e0fbb58f..54762eff3 100644 --- a/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java +++ b/examples/ip-intelligence/web/getting-started.cloud/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java @@ -59,6 +59,7 @@ package fiftyone.ipintelligence.examples.web; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.ipintelligence.examples.shared.KeyHelper; import fiftyone.ipintelligence.shared.IPIntelligenceData; import fiftyone.pipeline.core.data.FlowData; @@ -95,11 +96,21 @@ *

    * The configuration file for the pipeline is at * src/main/webapp/WEB-INF/51Degrees-Cloud.xml. The resource key is taken from the - * environment variable or system property "TestResourceKey" unless supplied as a command + * environment variable or system property "_51DEGREES_RESOURCE_KEY" unless supplied as a command * line argument. */ public class GettingStartedWebCloud extends HttpServlet { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "web-getting-started.cloud-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedwebcloud.java"; + private static final long serialVersionUID = 1734154705981153542L; + /** + * The port the example listens on. + */ + public static final int DEFAULT_PORT = 5151; public static Logger logger = LoggerFactory.getLogger(GettingStartedWebCloud.class); public static void main(String[] args) throws Exception { @@ -107,12 +118,20 @@ public static void main(String[] args) throws Exception { logger.info("Running Example {}", GettingStartedWebCloud.class); // Get the resource key from the command line, environment variable or - // system property. This sets the "TestResourceKey" system property + // system property. This sets the "_51DEGREES_RESOURCE_KEY" system property // which is interpolated in the pipeline configuration file. - KeyHelper.getOrSetTestResourceKey(args.length > 0 ? args[0] : null, true); + String resourceKey = KeyHelper.findResourceKey( + args.length > 0 ? args[0] : null); + if (resourceKey == null) { + KeyHelper.writeNoResourceKey(System.out, EXAMPLE_TAG); + FindOutMore.write(System.out, EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES); + return; + } + KeyHelper.getOrSetTestResourceKey(resourceKey, true); // start Jetty with this WebApp - EmbedJetty.runWebApp(getResourceBase(), 8083); + EmbedJetty.runWebApp(getResourceBase(), DEFAULT_PORT); } public static String getResourceBase() { @@ -140,6 +159,9 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re // The lookup has already been carried out by the PipelineFilter // which closes the FlowData at the end of the request, so it is not // closed here. + // A Flow Data cannot be used by two threads at once. Work handed to + // another thread or task gets the values it needs, read here on the + // request's thread, not the Flow Data. FlowData flowData = flowDataProvider.getFlowData(request); // Determine target IP for display. @@ -203,6 +225,8 @@ private String substituteTemplateValues(String template, IPIntelligenceData ipiD .replace("${AREAS_JS}", escapeForJs(asWktStringProperty(tryGet(ipiData::getAreas)))) .replace("${ACCURACY_RADIUS}", asIntegerProperty(tryGet(ipiData::getAccuracyRadiusMin))) .replace("${TIME_ZONE_OFFSET}", asIntegerProperty(tryGet(ipiData::getTimeZoneOffset))) + .replace("${FIND_OUT_MORE}", FindOutMore.html(EXAMPLE_TAG, + FindOutMore.CLOUD_REPOSITORIES)) .replace("${EVIDENCE_ROWS}", buildEvidenceRows(flowData)); } diff --git a/examples/ip-intelligence/web/getting-started.cloud/src/main/webapp/WEB-INF/51Degrees-Cloud.xml b/examples/ip-intelligence/web/getting-started.cloud/src/main/webapp/WEB-INF/51Degrees-Cloud.xml index 762f09ff1..bc1307a6e 100644 --- a/examples/ip-intelligence/web/getting-started.cloud/src/main/webapp/WEB-INF/51Degrees-Cloud.xml +++ b/examples/ip-intelligence/web/getting-started.cloud/src/main/webapp/WEB-INF/51Degrees-Cloud.xml @@ -27,9 +27,9 @@ - ${TestResourceKey} + ${_51DEGREES_RESOURCE_KEY} CloudRequestEngine diff --git a/examples/ip-intelligence/web/getting-started.cloud/src/main/webapp/WEB-INF/html/index.html b/examples/ip-intelligence/web/getting-started.cloud/src/main/webapp/WEB-INF/html/index.html index 05e1d4298..c2208d964 100644 --- a/examples/ip-intelligence/web/getting-started.cloud/src/main/webapp/WEB-INF/html/index.html +++ b/examples/ip-intelligence/web/getting-started.cloud/src/main/webapp/WEB-INF/html/index.html @@ -81,6 +81,8 @@

    Location Map

    + ${FIND_OUT_MORE} + @@ -114,9 +116,9 @@

    Location Map

    // Initialize the map var map = L.map('map'); - // Add CartoDB Positron tile layer - shows all place names in English - L.tileLayer('https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png', { - attribution: '© OpenStreetMap contributors © CARTO', + // Add the OpenStreetMap standard tile layer + L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', { + attribution: '© OpenStreetMap contributors', subdomains: 'abcd', maxZoom: 19 }).addTo(map); @@ -166,8 +168,8 @@

    Location Map

    if (!isNaN(lat) && !isNaN(lng)) { var map = L.map('map').setView([lat, lng], 10); - L.tileLayer('https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png', { - attribution: '© OpenStreetMap contributors © CARTO', + L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', { + attribution: '© OpenStreetMap contributors', subdomains: 'abcd', maxZoom: 19 }).addTo(map); diff --git a/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/DefaultPortTest.java b/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/DefaultPortTest.java new file mode 100644 index 000000000..63640a1ac --- /dev/null +++ b/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/DefaultPortTest.java @@ -0,0 +1,38 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.web; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Checks that the example listens on port 5151, as the readme says. It + * needs no resource key, so it runs where the page test is skipped. + */ +public class DefaultPortTest { + @Test + public void listensOn5151ByDefault() { + assertEquals(5151, GettingStartedWebCloud.DEFAULT_PORT); + } +} diff --git a/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudTest.java b/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudTest.java index 06a7b8b48..0eb321002 100644 --- a/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudTest.java +++ b/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudTest.java @@ -22,6 +22,7 @@ package fiftyone.ipintelligence.examples.web; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.ipintelligence.examples.shared.KeyHelper; import org.eclipse.jetty.server.Server; import org.junit.AfterClass; @@ -91,6 +92,9 @@ public void testWebCloud() throws Exception { // Assert we got a successful response assertEquals("Expected HTTP 200 OK response", 200, code); assertTrue("Response should not be empty", responseBody.length() > 0); + assertTrue("The page should end with the '" + FindOutMore.HEADING + + "' section", responseBody.contains( + "

    " + FindOutMore.HEADING + "

    ")); } @AfterClass diff --git a/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudWithoutKeyTest.java b/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudWithoutKeyTest.java new file mode 100644 index 000000000..32488acfc --- /dev/null +++ b/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudWithoutKeyTest.java @@ -0,0 +1,67 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.web; + +import fiftyone.ipintelligence.examples.shared.ExampleProcess; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Runs the IP Intelligence cloud web example as it runs from a fresh clone, + * with no resource key anywhere, and checks that it ends without an error, + * names the variable to set and links to where a key is created, then ends + * with its Find out more section. + */ +public class GettingStartedWebCloudWithoutKeyTest { + + private static final String CONFIGURE = "https://51degrees.com/configure/"; + + private static final String TAGS = + "?utm_source=code&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content="; + + private static void runWithoutKey(Class example) throws Exception { + ExampleProcess.Result result = ExampleProcess.runWithoutKeys(example); + System.out.println(result.output); + String output = result.output; + assertEquals(example.getSimpleName() + " should end with exit code 0", + 0, result.exitCode); + assertTrue("The output should name _51DEGREES_RESOURCE_KEY", + output.contains("Set the _51DEGREES_RESOURCE_KEY " + + "environment variable")); + int link = output.indexOf(CONFIGURE); + assertTrue("The output should link to where a key is created", + link >= 0 && output.startsWith(TAGS, + output.indexOf('?', link))); + assertTrue("The output should end with the Find out more section " + + "after the link", + output.lastIndexOf("Find out more") > link); + } + + @Test + public void gettingStartedWebCloud() throws Exception { + runWithoutKey(GettingStartedWebCloud.class); + } +} diff --git a/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/MapAttributionTest.java b/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/MapAttributionTest.java new file mode 100644 index 000000000..2c639011e --- /dev/null +++ b/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/MapAttributionTest.java @@ -0,0 +1,82 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.web; + +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * The page's map draws OpenStreetMap's standard tiles, so it credits + * OpenStreetMap as the other IP Intelligence web examples do, and says + * nothing of a tile provider it no longer uses. + */ +public class MapAttributionTest { + private static final String PAGE = + "src/main/webapp/WEB-INF/html/index.html"; + private static final String TILES = + "https://tile.openstreetmap.org/{z}/{x}/{y}.png"; + private static final String CREDIT = + "© " + + "OpenStreetMap contributors"; + + private static String page() throws Exception { + return new String( + Files.readAllBytes(Paths.get(PAGE)), + StandardCharsets.UTF_8); + } + + private static int count(String text, String part) { + int found = 0; + for (int at = text.indexOf(part); at >= 0; + at = text.indexOf(part, at + part.length())) { + found++; + } + return found; + } + + @Test + public void everyMapCreditsOpenStreetMapForItsTiles() throws Exception { + String page = page(); + int maps = count(page, TILES); + assertTrue("the page draws OpenStreetMap tiles", maps > 0); + assertEquals( + "each OpenStreetMap tile layer credits OpenStreetMap", + maps, + count(page, "attribution: '" + CREDIT + "'")); + } + + @Test + public void thePageNamesNoTileProviderItDoesNotUse() throws Exception { + String page = page(); + assertFalse("no CARTO credit", page.contains("CARTO")); + assertFalse("no CartoDB comment", page.contains("CartoDB")); + assertFalse("no CARTO tiles", page.contains("cartocdn.com")); + } +} diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java index 18f5bc05b..de9415bc5 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java +++ b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java @@ -47,6 +47,7 @@ package fiftyone.ipintelligence.examples.web; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.shared.DeviceData; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; @@ -88,7 +89,17 @@ * By default the free Lite data files are used. */ public class GettingStartedWebMixed extends HttpServlet { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "web-getting-started.mixed-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedwebmixed.java"; + private static final long serialVersionUID = 1734154705981153541L; + /** + * The port the example listens on. + */ + public static final int DEFAULT_PORT = 5151; public static Logger logger = LoggerFactory.getLogger(GettingStartedWebMixed.class); public static final String DD_DATA_FILE_DEFAULT = "device-detection/51Degrees-LiteV4.1.hash"; @@ -126,7 +137,7 @@ public static void main(String[] args) throws Exception { } // Start Jetty with this WebApp - EmbedJetty.runWebApp(getResourceBase(), 8081); + EmbedJetty.runWebApp(getResourceBase(), DEFAULT_PORT); } public static String getResourceBase() { @@ -147,6 +158,9 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re // The detection has already been carried out by the PipelineFilter, // which closes the FlowData at the end of the request, so it is not // closed here. + // A Flow Data cannot be used by two threads at once. Work handed to + // another thread or task gets the values it needs, read here on the + // request's thread, not the Flow Data. FlowData flowData = flowDataProvider.getFlowData(request); // Determine target IP for display. @@ -228,6 +242,8 @@ private String substituteTemplateValues(String template, DeviceData deviceData, escapeForJs(asString(ipiData.getAreas())) : "") .replace("${ACCURACY_RADIUS}", value(ipiEngine, "AccuracyRadiusMin", ipiData::getAccuracyRadiusMin)) .replace("${TIME_ZONE_OFFSET}", value(ipiEngine, "TimeZoneOffset", ipiData::getTimeZoneOffset)) + .replace("${FIND_OUT_MORE}", FindOutMore.html(EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES)) .replace("${EVIDENCE_ROWS}", buildEvidenceRows(flowData)) .replace("${LocationConfidence}", value(ipiEngine, "LocationConfidence", ipiData::getLocationConfidence)) .replace("${ConnectionType}", value(ipiEngine, "ConnectionType", ipiData::getConnectionType)) diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml b/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml index 83f577d3e..e720b2d5e 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml +++ b/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml @@ -36,7 +36,7 @@ false false - ${TestDataFile:-cxx/data/ip-intelligence/51Degrees-EnterpriseIpiV41.ipi} + ${TestDataFile:-cxx/data/ip-intelligence/51Degrees-LiteV41.ipi} LowMemory IPIntelligenceOnPremiseEngine diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/html/index.html b/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/html/index.html index 667dca9a9..e9af3cfff 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/html/index.html +++ b/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/html/index.html @@ -133,6 +133,7 @@

    Location map

    ${CONTACT_MESSAGE} + ${FIND_OUT_MORE} diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/js/examples.min.js b/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/js/examples.min.js index db9929460..4fbfed49c 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/js/examples.min.js +++ b/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/js/examples.min.js @@ -1 +1 @@ -(function(d){"use strict";var f={};f.bindDeviceCallback=function(e){if(e=e||{},!(typeof d.fod>"u"||!d.fod.complete)){var a=e.targetId||"content",r=e.fields||C;d.fod.complete(function(i){_(a,r(i))})}};function C(e){var a=e&&e.device||{},r=typeof a.hardwarename>"u"?"Unknown":[].concat(a.hardwarename).join(", ");return[["Hardware Name:",r],["Platform:",m(a.platformname,a.platformversion)],["Browser:",m(a.browsername,a.browserversion)],["Screen width (pixels):",a.screenpixelswidth],["Screen height (pixels):",a.screenpixelsheight],["Device Id:",a.deviceid]]}function m(e,a){return[e,a].filter(function(r){return r!=null&&r!==""}).join(" ")}function _(e,a){var r=document.getElementById(e);if(r){var i=document.createElement("table");i.className="c-eg-table";var t=document.createElement("tr");t.className="c-eg-table__row c-eg-table__head",o(t,"th","Key",!1),o(t,"th","Value",!1),i.appendChild(t),a.forEach(function(l){var n=document.createElement("tr");n.className="c-eg-table__row c-eg-table__row--used",o(n,"td",l[0],!0),o(n,"td",l[1],!1),i.appendChild(n)}),r.appendChild(i)}}function o(e,a,r,i){var t=document.createElement(a);t.className="c-eg-table__cell"+(i?" c-eg-table__cell--key":"");var l=document.createTextNode(r??"");if(i){var n=document.createElement("strong");n.appendChild(l),l=n}t.appendChild(l),e.appendChild(t)}f.initLocationMap=function(e){e=e||{};var a=d.L;if(a){var r=e.sectionId||"map-section",i=e.canvasId||"map",t=e.areasWkt||"",l=e.latitude,n=e.longitude,v=e.labels||{ipLocation:"IP Location",lat:"Lat",lng:"Lng"},h=e.tileUrl||"https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png",w={attribution:"\xA9 OpenStreetMap contributors \xA9 CARTO",subdomains:"abcd",maxZoom:19};if(c(t)&&t!=="POLYGON EMPTY"&&d.wellknown){u(r,!0);var s=a.map(i);a.tileLayer(h,w).addTo(s);try{var g=d.wellknown.parse(t);if(g){var E=a.geoJSON(g,{style:{color:"#CC2B27",weight:2,opacity:.8,fillColor:"#CC2B27",fillOpacity:.2}}).addTo(s);s.fitBounds(E.getBounds()),p(a,s,l,n,v)}}catch(k){console.error("Error parsing polygon:",k),u(r,!1)}}else if(c(l)&&c(n)){var b=parseFloat(l),N=parseFloat(n);if(!isNaN(b)&&!isNaN(N)){u(r,!0);var y=a.map(i).setView([b,N],10);a.tileLayer(h,w).addTo(y),p(a,y,l,n,v)}}}};function p(e,a,r,i,t){if(!(!c(r)||!c(i))){var l=parseFloat(r),n=parseFloat(i);isNaN(l)||isNaN(n)||e.marker([l,n]).addTo(a).bindPopup(t.ipLocation+"
    "+t.lat+": "+l+"
    "+t.lng+": "+n).openPopup()}}function u(e,a){var r=document.getElementById(e);r&&(r.style.display=a?"block":"none")}function c(e){return e&&String(e).indexOf("Unknown")===-1&&e!=="0"&&e!==""}d.fodExamples=f})(window); +(function(d){"use strict";var f={};f.bindDeviceCallback=function(e){if(e=e||{},!(typeof d.fod>"u"||!d.fod.complete)){var a=e.targetId||"content",r=e.fields||C;d.fod.complete(function(i){_(a,r(i))})}};function C(e){var a=e&&e.device||{},r=typeof a.hardwarename>"u"?"Unknown":[].concat(a.hardwarename).join(", ");return[["Hardware Name:",r],["Platform:",m(a.platformname,a.platformversion)],["Browser:",m(a.browsername,a.browserversion)],["Screen width (pixels):",a.screenpixelswidth],["Screen height (pixels):",a.screenpixelsheight],["Device Id:",a.deviceid]]}function m(e,a){return[e,a].filter(function(r){return r!=null&&r!==""}).join(" ")}function _(e,a){var r=document.getElementById(e);if(r){var i=document.createElement("table");i.className="c-eg-table";var t=document.createElement("tr");t.className="c-eg-table__row c-eg-table__head",o(t,"th","Key",!1),o(t,"th","Value",!1),i.appendChild(t),a.forEach(function(l){var n=document.createElement("tr");n.className="c-eg-table__row c-eg-table__row--used",o(n,"td",l[0],!0),o(n,"td",l[1],!1),i.appendChild(n)}),r.appendChild(i)}}function o(e,a,r,i){var t=document.createElement(a);t.className="c-eg-table__cell"+(i?" c-eg-table__cell--key":"");var l=document.createTextNode(r??"");if(i){var n=document.createElement("strong");n.appendChild(l),l=n}t.appendChild(l),e.appendChild(t)}f.initLocationMap=function(e){e=e||{};var a=d.L;if(a){var r=e.sectionId||"map-section",i=e.canvasId||"map",t=e.areasWkt||"",l=e.latitude,n=e.longitude,v=e.labels||{ipLocation:"IP Location",lat:"Lat",lng:"Lng"},h=e.tileUrl||"https://tile.openstreetmap.org/{z}/{x}/{y}.png",w={attribution:"© OpenStreetMap contributors",subdomains:"abcd",maxZoom:19};if(c(t)&&t!=="POLYGON EMPTY"&&d.wellknown){u(r,!0);var s=a.map(i);a.tileLayer(h,w).addTo(s);try{var g=d.wellknown.parse(t);if(g){var E=a.geoJSON(g,{style:{color:"#CC2B27",weight:2,opacity:.8,fillColor:"#CC2B27",fillOpacity:.2}}).addTo(s);s.fitBounds(E.getBounds()),p(a,s,l,n,v)}}catch(k){console.error("Error parsing polygon:",k),u(r,!1)}}else if(c(l)&&c(n)){var b=parseFloat(l),N=parseFloat(n);if(!isNaN(b)&&!isNaN(N)){u(r,!0);var y=a.map(i).setView([b,N],10);a.tileLayer(h,w).addTo(y),p(a,y,l,n,v)}}}};function p(e,a,r,i,t){if(!(!c(r)||!c(i))){var l=parseFloat(r),n=parseFloat(i);isNaN(l)||isNaN(n)||e.marker([l,n]).addTo(a).bindPopup(t.ipLocation+"
    "+t.lat+": "+l+"
    "+t.lng+": "+n).openPopup()}}function u(e,a){var r=document.getElementById(e);r&&(r.style.display=a?"block":"none")}function c(e){return e&&String(e).indexOf("Unknown")===-1&&e!=="0"&&e!==""}d.fodExamples=f})(window); diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/test/java/fiftyone/ipintelligence/examples/web/DefaultPortTest.java b/examples/ip-intelligence/web/getting-started.mixed/src/test/java/fiftyone/ipintelligence/examples/web/DefaultPortTest.java new file mode 100644 index 000000000..31513a79f --- /dev/null +++ b/examples/ip-intelligence/web/getting-started.mixed/src/test/java/fiftyone/ipintelligence/examples/web/DefaultPortTest.java @@ -0,0 +1,38 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.web; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Checks that the example listens on port 5151, as the readme says. It + * needs no data file, so it runs where the page test is skipped. + */ +public class DefaultPortTest { + @Test + public void listensOn5151ByDefault() { + assertEquals(5151, GettingStartedWebMixed.DEFAULT_PORT); + } +} diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixedTest.java b/examples/ip-intelligence/web/getting-started.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixedTest.java index 4aa023bb3..276a418b9 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixedTest.java +++ b/examples/ip-intelligence/web/getting-started.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixedTest.java @@ -22,6 +22,7 @@ package fiftyone.ipintelligence.examples.web; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.ipintelligence.examples.shared.DataFileHelper; import org.eclipse.jetty.server.Server; import org.junit.AfterClass; @@ -117,6 +118,9 @@ public void testWebMixed() throws Exception { assertEquals("Expected HTTP 200 OK response", 200, code); assertTrue("Response should not be empty", responseBody.length() > 0); + assertTrue("The page should end with the '" + FindOutMore.HEADING + + "' section", responseBody.contains( + "

    " + FindOutMore.HEADING + "

    ")); } @AfterClass diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/test/java/fiftyone/ipintelligence/examples/web/OptionsFileTest.java b/examples/ip-intelligence/web/getting-started.mixed/src/test/java/fiftyone/ipintelligence/examples/web/OptionsFileTest.java new file mode 100644 index 000000000..87685c702 --- /dev/null +++ b/examples/ip-intelligence/web/getting-started.mixed/src/test/java/fiftyone/ipintelligence/examples/web/OptionsFileTest.java @@ -0,0 +1,61 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.web; + +import fiftyone.ipintelligence.examples.shared.DataFileHelper; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Checks the options file this web example runs with. + */ +public class OptionsFileTest { + + /** + * With no data file set, the example falls back to the free Lite file, + * which a fresh clone downloads, and not to the Enterprise file, which a + * fresh clone never has. + */ + @Test + public void defaultDataFile_FreeLiteFile() throws IOException { + String xml = new String(Files.readAllBytes(Paths.get( + System.getProperty("basedir", "."), "src", "main", "webapp", + "WEB-INF", "51Degrees-Mixed.xml")), StandardCharsets.UTF_8); + Matcher dataFile = Pattern.compile( + "\\$\\{TestDataFile:-([^}]*)\\}") + .matcher(xml); + assertTrue("51Degrees-Mixed.xml should give a default IP Intelligence " + + "data file", dataFile.find()); + assertEquals("cxx/data/" + DataFileHelper.LITE_DATA_FILE_REL_PATH, + dataFile.group(1)); + } +} diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java b/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java index d87b20d5d..ff8c0e45f 100644 --- a/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java +++ b/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java @@ -61,6 +61,7 @@ package fiftyone.ipintelligence.examples.web; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; import fiftyone.ipintelligence.examples.shared.DataFileHelper; import fiftyone.ipintelligence.shared.IPIntelligenceData; @@ -107,7 +108,17 @@ * obtain an Enterprise data file, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=web-getting-started.onprem-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedwebonprem.java&utm_term=gettingstartedwebonprem). */ public class GettingStartedWebOnPrem extends HttpServlet { + /** + * This example's tag in the links of its "Find out more" section. + */ + private static final String EXAMPLE_TAG = + "web-getting-started.onprem-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedwebonprem.java"; + private static final long serialVersionUID = 1734154705981153540L; + /** + * The port the example listens on. + */ + public static final int DEFAULT_PORT = 5151; public static Logger logger = LoggerFactory.getLogger(GettingStartedWebOnPrem.class); public static void main(String[] args) throws Exception { @@ -131,7 +142,7 @@ public static void main(String[] args) throws Exception { } // start Jetty with this WebApp - EmbedJetty.runWebApp(getResourceBase(), 8082); + EmbedJetty.runWebApp(getResourceBase(), DEFAULT_PORT); } public static String getResourceBase() { @@ -159,6 +170,9 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re // The lookup has already been carried out by the PipelineFilter // which closes the FlowData at the end of the request, so it is not // closed here. + // A Flow Data cannot be used by two threads at once. Work handed to + // another thread or task gets the values it needs, read here on the + // request's thread, not the Flow Data. FlowData flowData = flowDataProvider.getFlowData(request); // Determine target IP for display. @@ -227,6 +241,8 @@ private String substituteTemplateValues(String template, IPIntelligenceData ipiD escapeForJs(asString(ipiData.getAreas())) : "") .replace("${ACCURACY_RADIUS}", value(ipiEngine, "AccuracyRadiusMin", ipiData::getAccuracyRadiusMin)) .replace("${TIME_ZONE_OFFSET}", value(ipiEngine, "TimeZoneOffset", ipiData::getTimeZoneOffset)) + .replace("${FIND_OUT_MORE}", FindOutMore.html(EXAMPLE_TAG, + FindOutMore.ON_PREMISE_REPOSITORIES)) .replace("${EVIDENCE_ROWS}", buildEvidenceRows(flowData)) // On-premise engine: only invite the user to contact us about the // Enterprise data file when running on a free data file. diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml b/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml index c8e8d515b..36adb9023 100644 --- a/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml +++ b/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml @@ -29,7 +29,7 @@ false - ${TestDataFile:-cxx/data/ip-intelligence/51Degrees-EnterpriseIpiV41.ipi} + ${TestDataFile:-cxx/data/ip-intelligence/51Degrees-LiteV41.ipi} LowMemory
    IPIntelligenceOnPremiseEngine diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/html/index.html b/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/html/index.html index 2abdface2..afb91cab8 100644 --- a/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/html/index.html +++ b/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/WEB-INF/html/index.html @@ -92,6 +92,7 @@

    Location map

    ${CONTACT_MESSAGE} + ${FIND_OUT_MORE} diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/js/examples.min.js b/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/js/examples.min.js index db9929460..4fbfed49c 100644 --- a/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/js/examples.min.js +++ b/examples/ip-intelligence/web/getting-started.onprem/src/main/webapp/js/examples.min.js @@ -1 +1 @@ -(function(d){"use strict";var f={};f.bindDeviceCallback=function(e){if(e=e||{},!(typeof d.fod>"u"||!d.fod.complete)){var a=e.targetId||"content",r=e.fields||C;d.fod.complete(function(i){_(a,r(i))})}};function C(e){var a=e&&e.device||{},r=typeof a.hardwarename>"u"?"Unknown":[].concat(a.hardwarename).join(", ");return[["Hardware Name:",r],["Platform:",m(a.platformname,a.platformversion)],["Browser:",m(a.browsername,a.browserversion)],["Screen width (pixels):",a.screenpixelswidth],["Screen height (pixels):",a.screenpixelsheight],["Device Id:",a.deviceid]]}function m(e,a){return[e,a].filter(function(r){return r!=null&&r!==""}).join(" ")}function _(e,a){var r=document.getElementById(e);if(r){var i=document.createElement("table");i.className="c-eg-table";var t=document.createElement("tr");t.className="c-eg-table__row c-eg-table__head",o(t,"th","Key",!1),o(t,"th","Value",!1),i.appendChild(t),a.forEach(function(l){var n=document.createElement("tr");n.className="c-eg-table__row c-eg-table__row--used",o(n,"td",l[0],!0),o(n,"td",l[1],!1),i.appendChild(n)}),r.appendChild(i)}}function o(e,a,r,i){var t=document.createElement(a);t.className="c-eg-table__cell"+(i?" c-eg-table__cell--key":"");var l=document.createTextNode(r??"");if(i){var n=document.createElement("strong");n.appendChild(l),l=n}t.appendChild(l),e.appendChild(t)}f.initLocationMap=function(e){e=e||{};var a=d.L;if(a){var r=e.sectionId||"map-section",i=e.canvasId||"map",t=e.areasWkt||"",l=e.latitude,n=e.longitude,v=e.labels||{ipLocation:"IP Location",lat:"Lat",lng:"Lng"},h=e.tileUrl||"https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png",w={attribution:"\xA9 OpenStreetMap contributors \xA9 CARTO",subdomains:"abcd",maxZoom:19};if(c(t)&&t!=="POLYGON EMPTY"&&d.wellknown){u(r,!0);var s=a.map(i);a.tileLayer(h,w).addTo(s);try{var g=d.wellknown.parse(t);if(g){var E=a.geoJSON(g,{style:{color:"#CC2B27",weight:2,opacity:.8,fillColor:"#CC2B27",fillOpacity:.2}}).addTo(s);s.fitBounds(E.getBounds()),p(a,s,l,n,v)}}catch(k){console.error("Error parsing polygon:",k),u(r,!1)}}else if(c(l)&&c(n)){var b=parseFloat(l),N=parseFloat(n);if(!isNaN(b)&&!isNaN(N)){u(r,!0);var y=a.map(i).setView([b,N],10);a.tileLayer(h,w).addTo(y),p(a,y,l,n,v)}}}};function p(e,a,r,i,t){if(!(!c(r)||!c(i))){var l=parseFloat(r),n=parseFloat(i);isNaN(l)||isNaN(n)||e.marker([l,n]).addTo(a).bindPopup(t.ipLocation+"
    "+t.lat+": "+l+"
    "+t.lng+": "+n).openPopup()}}function u(e,a){var r=document.getElementById(e);r&&(r.style.display=a?"block":"none")}function c(e){return e&&String(e).indexOf("Unknown")===-1&&e!=="0"&&e!==""}d.fodExamples=f})(window); +(function(d){"use strict";var f={};f.bindDeviceCallback=function(e){if(e=e||{},!(typeof d.fod>"u"||!d.fod.complete)){var a=e.targetId||"content",r=e.fields||C;d.fod.complete(function(i){_(a,r(i))})}};function C(e){var a=e&&e.device||{},r=typeof a.hardwarename>"u"?"Unknown":[].concat(a.hardwarename).join(", ");return[["Hardware Name:",r],["Platform:",m(a.platformname,a.platformversion)],["Browser:",m(a.browsername,a.browserversion)],["Screen width (pixels):",a.screenpixelswidth],["Screen height (pixels):",a.screenpixelsheight],["Device Id:",a.deviceid]]}function m(e,a){return[e,a].filter(function(r){return r!=null&&r!==""}).join(" ")}function _(e,a){var r=document.getElementById(e);if(r){var i=document.createElement("table");i.className="c-eg-table";var t=document.createElement("tr");t.className="c-eg-table__row c-eg-table__head",o(t,"th","Key",!1),o(t,"th","Value",!1),i.appendChild(t),a.forEach(function(l){var n=document.createElement("tr");n.className="c-eg-table__row c-eg-table__row--used",o(n,"td",l[0],!0),o(n,"td",l[1],!1),i.appendChild(n)}),r.appendChild(i)}}function o(e,a,r,i){var t=document.createElement(a);t.className="c-eg-table__cell"+(i?" c-eg-table__cell--key":"");var l=document.createTextNode(r??"");if(i){var n=document.createElement("strong");n.appendChild(l),l=n}t.appendChild(l),e.appendChild(t)}f.initLocationMap=function(e){e=e||{};var a=d.L;if(a){var r=e.sectionId||"map-section",i=e.canvasId||"map",t=e.areasWkt||"",l=e.latitude,n=e.longitude,v=e.labels||{ipLocation:"IP Location",lat:"Lat",lng:"Lng"},h=e.tileUrl||"https://tile.openstreetmap.org/{z}/{x}/{y}.png",w={attribution:"© OpenStreetMap contributors",subdomains:"abcd",maxZoom:19};if(c(t)&&t!=="POLYGON EMPTY"&&d.wellknown){u(r,!0);var s=a.map(i);a.tileLayer(h,w).addTo(s);try{var g=d.wellknown.parse(t);if(g){var E=a.geoJSON(g,{style:{color:"#CC2B27",weight:2,opacity:.8,fillColor:"#CC2B27",fillOpacity:.2}}).addTo(s);s.fitBounds(E.getBounds()),p(a,s,l,n,v)}}catch(k){console.error("Error parsing polygon:",k),u(r,!1)}}else if(c(l)&&c(n)){var b=parseFloat(l),N=parseFloat(n);if(!isNaN(b)&&!isNaN(N)){u(r,!0);var y=a.map(i).setView([b,N],10);a.tileLayer(h,w).addTo(y),p(a,y,l,n,v)}}}};function p(e,a,r,i,t){if(!(!c(r)||!c(i))){var l=parseFloat(r),n=parseFloat(i);isNaN(l)||isNaN(n)||e.marker([l,n]).addTo(a).bindPopup(t.ipLocation+"
    "+t.lat+": "+l+"
    "+t.lng+": "+n).openPopup()}}function u(e,a){var r=document.getElementById(e);r&&(r.style.display=a?"block":"none")}function c(e){return e&&String(e).indexOf("Unknown")===-1&&e!=="0"&&e!==""}d.fodExamples=f})(window); diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/DefaultPortTest.java b/examples/ip-intelligence/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/DefaultPortTest.java new file mode 100644 index 000000000..074b70592 --- /dev/null +++ b/examples/ip-intelligence/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/DefaultPortTest.java @@ -0,0 +1,38 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.web; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Checks that the example listens on port 5151, as the readme says. It + * needs no data file, so it runs where the page test is skipped. + */ +public class DefaultPortTest { + @Test + public void listensOn5151ByDefault() { + assertEquals(5151, GettingStartedWebOnPrem.DEFAULT_PORT); + } +} diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPremTest.java b/examples/ip-intelligence/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPremTest.java index e1831b39b..f995a4ea6 100644 --- a/examples/ip-intelligence/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPremTest.java +++ b/examples/ip-intelligence/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPremTest.java @@ -22,6 +22,7 @@ package fiftyone.ipintelligence.examples.web; +import fiftyone.ipintelligence.examples.shared.FindOutMore; import fiftyone.ipintelligence.examples.shared.DataFileHelper; import org.eclipse.jetty.server.Server; import org.junit.AfterClass; @@ -96,6 +97,9 @@ public void testWebOnPrem() throws Exception { // Assert we got a successful response assertEquals("Expected HTTP 200 OK response", 200, code); assertTrue("Response should not be empty", responseBody.length() > 0); + assertTrue("The page should end with the '" + FindOutMore.HEADING + + "' section", responseBody.contains( + "

    " + FindOutMore.HEADING + "

    ")); } @AfterClass diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/OptionsFileTest.java b/examples/ip-intelligence/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/OptionsFileTest.java new file mode 100644 index 000000000..047f34bdc --- /dev/null +++ b/examples/ip-intelligence/web/getting-started.onprem/src/test/java/fiftyone/ipintelligence/examples/web/OptionsFileTest.java @@ -0,0 +1,61 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.web; + +import fiftyone.ipintelligence.examples.shared.DataFileHelper; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Checks the options file this web example runs with. + */ +public class OptionsFileTest { + + /** + * With no data file set, the example falls back to the free Lite file, + * which a fresh clone downloads, and not to the Enterprise file, which a + * fresh clone never has. + */ + @Test + public void defaultDataFile_FreeLiteFile() throws IOException { + String xml = new String(Files.readAllBytes(Paths.get( + System.getProperty("basedir", "."), "src", "main", "webapp", + "WEB-INF", "51Degrees-OnPrem.xml")), StandardCharsets.UTF_8); + Matcher dataFile = Pattern.compile( + "\\$\\{TestDataFile:-([^}]*)\\}") + .matcher(xml); + assertTrue("51Degrees-OnPrem.xml should give a default IP Intelligence " + + "data file", dataFile.find()); + assertEquals("cxx/data/" + DataFileHelper.LITE_DATA_FILE_REL_PATH, + dataFile.group(1)); + } +} diff --git a/ip-intelligence/README.md b/ip-intelligence/README.md index e5c19598b..abfcd1385 100644 --- a/ip-intelligence/README.md +++ b/ip-intelligence/README.md @@ -24,7 +24,7 @@ The cloud service for IP Intelligence is currently in development. You will require [resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon) to use the Cloud API, as described on our website. Get resource keys from -our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon) on +our [configuration page](https://51degrees.com/configure/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=cloud-coming-soon) on how to use this. #### On-Premise @@ -110,20 +110,18 @@ repository. You will need [resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=tests) (see above) to complete the tests and run examples which include exercising the cloud API. -The tests look for a resource key in the following order: - -1. The aligned `_51DEGREES_RESOURCE_KEY` name, set as an environment variable - or as a system property. -2. The legacy `TestResourceKey` name, which is still supported and is checked - after the aligned name. +The tests read a resource key from `_51DEGREES_RESOURCE_KEY`, set as an +environment variable or as a system property, and no other name. A cloud test +with no key is skipped. To verify the code: ``` -mvn clean test -DTestResourceKey=[Resource Key] +mvn clean test -D_51DEGREES_RESOURCE_KEY=[Resource Key] ``` -For tests and examples that require a license key add the following option: -- `-DLicenseKey=[License Key]` +For tests and examples that require a license key, set +`_51DEGREES_LICENSE_KEY` or add the following option: +- `-D_51DEGREES_LICENSE_KEY=[License Key]` ## Projects diff --git a/ip-intelligence/ip-intelligence.cloud/pom.xml b/ip-intelligence/ip-intelligence.cloud/pom.xml index cfc6ffa32..178fbcff0 100644 --- a/ip-intelligence/ip-intelligence.cloud/pom.xml +++ b/ip-intelligence/ip-intelligence.cloud/pom.xml @@ -11,7 +11,7 @@ ip-intelligence.cloud 51Degrees :: IP Intelligence :: Cloud - Provides IP Intelligence for the 51Degrees Pipeline API using the 51Degrees Cloud service. This can be used as an alternative to legacy IP Intelligence services such as MaxMind or IPinfo. + IP Intelligence and IP geolocation for Java, using the 51Degrees cloud service. Finds an IP address's likely location (country, region, town, and coordinates), connection type, and network. Needs a Resource Key that says which properties to return. Built on the 51Degrees Pipeline API. An alternative to MaxMind and IPinfo. https://github.com/51Degrees/51degrees-java diff --git a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/data/MultiIPIDataCloud.java b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/data/MultiIPIDataCloud.java index f25e0edc6..936fcb0e2 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/data/MultiIPIDataCloud.java +++ b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/data/MultiIPIDataCloud.java @@ -86,8 +86,12 @@ public List getProfiles() { return getProfileList(); } + /** + * @throws IllegalStateException if the Flow Element has finished + */ @Override public void addProfile(IPIntelligenceData profile) { + checkWriting(); getProfileList().add(profile); } diff --git a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java index 99b84cbb3..bd5e14367 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java +++ b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java @@ -23,7 +23,6 @@ package fiftyone.ipintelligence.cloud.flowelements; import fiftyone.ipintelligence.cloud.data.IPIntelligenceDataCloud; -import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; import fiftyone.pipeline.cloudrequestengine.flowelements.CloudAspectEngineBase; import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; import fiftyone.pipeline.core.data.AccessiblePropertyMetaData; @@ -32,7 +31,6 @@ import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.data.factories.ElementDataFactory; import fiftyone.pipeline.core.data.types.JavaScript; -import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.data.AspectPropertyMetaData; import fiftyone.pipeline.engines.data.AspectPropertyMetaDataDefault; @@ -49,6 +47,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -67,7 +66,7 @@ public class IPIntelligenceCloudEngine private String dataSourceTier; private final Object metadataLock = new Object(); // Lower-cased names of properties the cloud returns as weighted lists - // (cloud type "Weighted..."), so processEngine can read their value/weight + // (cloud type "Weighted..."), so processCloudEngine can read their value/weight // pairs rather than treating them as a plain list. private final Set weightedPropertyNames = new HashSet<>(); CloudRequestEngine cloudRequestEngine; @@ -135,111 +134,114 @@ public EvidenceKeyFilter getEvidenceKeyFilter() { return new EvidenceKeyFilterWhitelist(new ArrayList()); } + /** + * Populate the Element Data from this engine's part of the cloud + * response. The shared Cloud Aspect Engine processing finds the Cloud + * Request Engine's data, checks the request ran, and calls this only when + * there is a response. + * @param data the Flow Data being processed + * @param aspectData the Element Data to populate + * @param json the JSON response from the {@link CloudRequestEngine} + */ @Override - protected void processEngine(FlowData data, IPIntelligenceDataCloud aspectData) { - if (cloudRequestEngine == null) { - throw new PipelineConfigurationException( - "The '" + getClass().getName() + "' requires a " + - "'CloudRequestEngine' before it in the Pipeline. This " + - "engine will be unable to produce results until this is " + - "corrected."); - } - else { - CloudRequestData requestData = - data.getFromElement(cloudRequestEngine); - String json = ""; - json = requestData.getJsonResponse(); - - // Extract data from json to the aspectData instance. - JSONObject jsonObj = new JSONObject(json); - JSONObject ipiObj = getElementJson(jsonObj); - - Map propertyMap = - new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - - for (AspectPropertyMetaData property : getProperties()) { - if (weightedPropertyNames.contains( - property.getName().toLowerCase())) { + protected void processCloudEngine( + FlowData data, + IPIntelligenceDataCloud aspectData, + String json) { + JSONObject jsonObj = new JSONObject(json); + JSONObject ipiObj = getElementJson(jsonObj); + + Map propertyMap = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + + for (AspectPropertyMetaData property : getProperties()) { + // A property the response leaves out has no entry, so + // reading it throws a PropertyMissingException. + if (ipiObj.has( + property.getName().toLowerCase(Locale.ROOT)) == false) { + continue; + } + if (weightedPropertyNames.contains( + property.getName().toLowerCase(Locale.ROOT))) { + propertyMap.put( + property.getName(), + getWeightedListAspectPropertyValue(ipiObj, property)); + continue; + } + String type = property.getType().getSimpleName(); + switch(type) { + case ("List"): propertyMap.put( property.getName(), - getWeightedListAspectPropertyValue(ipiObj, property)); - continue; - } - String type = property.getType().getSimpleName(); - switch(type) { - case ("List"): - propertyMap.put( - property.getName(), - getListAspectPropertyValue(ipiObj, property)); - break; - case ("JavaScript"): - propertyMap.put( - property.getName(), - getJavaScriptAspectPropertyValue(ipiObj, property)); - break; - case ("String"): - propertyMap.put( - property.getName(), - getStringAspectPropertyValue(ipiObj, property)); - break; - case ("InetAddress"): - propertyMap.put( - property.getName(), - getInetAddressAspectPropertyValue(ipiObj, property)); - break; - case ("WktString"): - propertyMap.put( - property.getName(), - getWktStringAspectPropertyValue(ipiObj, property)); - break; - case ("boolean"): - propertyMap.put( - property.getName(), - getBooleanAspectPropertyValue(ipiObj, property)); - break; - case ("int"): - propertyMap.put( - property.getName(), - getIntegerAspectPropertyValue(ipiObj, property)); - break; - case ("double"): - propertyMap.put( - property.getName(), - getDoubleAspectPropertyValue(ipiObj, property)); + getListAspectPropertyValue(ipiObj, property)); + break; + case ("JavaScript"): + propertyMap.put( + property.getName(), + getJavaScriptAspectPropertyValue(ipiObj, property)); + break; + case ("String"): + propertyMap.put( + property.getName(), + getStringAspectPropertyValue(ipiObj, property)); break; - case ("Float"): - propertyMap.put( - property.getName(), - getFloatAspectPropertyValue(ipiObj, property)); + case ("InetAddress"): + propertyMap.put( + property.getName(), + getInetAddressAspectPropertyValue(ipiObj, property)); + break; + case ("WktString"): + propertyMap.put( + property.getName(), + getWktStringAspectPropertyValue(ipiObj, property)); + break; + case ("boolean"): + propertyMap.put( + property.getName(), + getBooleanAspectPropertyValue(ipiObj, property)); + break; + case ("int"): + propertyMap.put( + property.getName(), + getIntegerAspectPropertyValue(ipiObj, property)); + break; + case ("double"): + propertyMap.put( + property.getName(), + getDoubleAspectPropertyValue(ipiObj, property)); + break; + case ("Float"): + propertyMap.put( + property.getName(), + getFloatAspectPropertyValue(ipiObj, property)); + break; + default: + propertyMap.put( + property.getName(), + getStringAspectPropertyValue(ipiObj, property)); break; - default: - propertyMap.put( - property.getName(), - getStringAspectPropertyValue(ipiObj, property)); - break; - } } + } - // Add no value messages for any properties which have not yet been - // populated. - for (String key : ipiObj.keySet()) { - if (key.endsWith("nullreason")) { - String actualKey = key - .replace("nullreason", ""); - if (propertyMap.containsKey(actualKey) == false) { - AspectPropertyValue nullValue = - new AspectPropertyValueDefault(); - nullValue.setNoValueMessage( - ipiObj.optString(key, "Unknown")); - propertyMap.put( - actualKey, - nullValue); - } + // Add no value messages for any properties which have not yet been + // populated. + for (String key : ipiObj.keySet()) { + if (key.endsWith("nullreason")) { + String actualKey = key + .replace("nullreason", ""); + if (propertyMap.containsKey(actualKey) == false) { + AspectPropertyValue nullValue = + new AspectPropertyValueDefault(); + nullValue.setNoValueMessage( + ipiObj.optString(key, "Unknown")); + propertyMap.put( + actualKey, + nullValue); } } - - aspectData.populateFromMap(propertyMap); } + + aspectData.populateFromMap(propertyMap); } @Override @@ -280,7 +282,7 @@ private boolean loadAspectProperties(CloudRequestEngine engine) { for (AccessiblePropertyMetaData.PropertyMetaData item : product.properties) { if (item.type != null && item.type.startsWith("Weighted")) { - weightedPropertyNames.add(item.name.toLowerCase()); + weightedPropertyNames.add(item.name.toLowerCase(Locale.ROOT)); } AspectPropertyMetaData property = new AspectPropertyMetaDataDefault( item.name, @@ -355,7 +357,7 @@ private Class resolvePropertyType( private AspectPropertyValue getIntegerAspectPropertyValue( JSONObject ipiObj, AspectPropertyMetaData property) { - String key = property.getName().toLowerCase(); + String key = property.getName().toLowerCase(Locale.ROOT); AspectPropertyValue intValue = new AspectPropertyValueDefault<>(); if(ipiObj.isNull(key)){ intValue.setNoValueMessage(getNoValueReason(ipiObj, key)); @@ -367,7 +369,7 @@ private AspectPropertyValue getIntegerAspectPropertyValue( } private AspectPropertyValue getDoubleAspectPropertyValue(JSONObject ipiObj, AspectPropertyMetaData property) { - String key = property.getName().toLowerCase(); + String key = property.getName().toLowerCase(Locale.ROOT); AspectPropertyValue doubleValue = new AspectPropertyValueDefault<>(); if(ipiObj.isNull(key)){ doubleValue.setNoValueMessage(getNoValueReason(ipiObj, key)); @@ -388,7 +390,7 @@ private AspectPropertyValue getDoubleAspectPropertyValue(JSONObject ipiO * for the value not being present */ private AspectPropertyValue getFloatAspectPropertyValue(JSONObject ipiObj, AspectPropertyMetaData property) { - String key = property.getName().toLowerCase(); + String key = property.getName().toLowerCase(Locale.ROOT); AspectPropertyValue floatValue = new AspectPropertyValueDefault<>(); if(ipiObj.isNull(key)){ floatValue.setNoValueMessage(getNoValueReason(ipiObj, key)); @@ -410,7 +412,7 @@ private AspectPropertyValue getFloatAspectPropertyValue(JSONObject ipiObj private AspectPropertyValue getBooleanAspectPropertyValue( JSONObject ipiObj, AspectPropertyMetaData property) { - String key = property.getName().toLowerCase(); + String key = property.getName().toLowerCase(Locale.ROOT); AspectPropertyValue booleanValue = new AspectPropertyValueDefault<>(); if(ipiObj.isNull(key)){ booleanValue.setNoValueMessage(getNoValueReason(ipiObj, key)); @@ -432,7 +434,7 @@ private AspectPropertyValue getBooleanAspectPropertyValue( private AspectPropertyValue> getListAspectPropertyValue( JSONObject ipiObj, AspectPropertyMetaData property){ - String key = property.getName().toLowerCase(); + String key = property.getName().toLowerCase(Locale.ROOT); AspectPropertyValue> listValue = new AspectPropertyValueDefault<>(); if(ipiObj.isNull(key)) { @@ -465,7 +467,7 @@ private AspectPropertyValue> getListAspectPropertyValue( getWeightedListAspectPropertyValue( JSONObject deviceObj, AspectPropertyMetaData property) { - String key = property.getName().toLowerCase(); + String key = property.getName().toLowerCase(Locale.ROOT); AspectPropertyValue>> weightedValue = new AspectPropertyValueDefault<>(); if (deviceObj.isNull(key)) { @@ -500,7 +502,7 @@ private AspectPropertyValue> getListAspectPropertyValue( private AspectPropertyValue getInetAddressAspectPropertyValue( JSONObject deviceObj, AspectPropertyMetaData property) { - String key = property.getName().toLowerCase(); + String key = property.getName().toLowerCase(Locale.ROOT); AspectPropertyValue ipValue = new AspectPropertyValueDefault<>(); if (deviceObj.isNull(key)) { ipValue.setNoValueMessage(getNoValueReason(deviceObj, key)); @@ -526,7 +528,7 @@ private AspectPropertyValue getInetAddressAspectPropertyValue( private AspectPropertyValue getJavaScriptAspectPropertyValue( JSONObject ipiObj, AspectPropertyMetaData property){ - String key = property.getName().toLowerCase(); + String key = property.getName().toLowerCase(Locale.ROOT); AspectPropertyValue jsValue = new AspectPropertyValueDefault<>(); if(ipiObj.isNull(key)){ jsValue.setNoValueMessage(getNoValueReason(ipiObj, key)); @@ -551,7 +553,7 @@ private AspectPropertyValue getJavaScriptAspectPropertyValue( private AspectPropertyValue getWktStringAspectPropertyValue( JSONObject ipiObj, AspectPropertyMetaData property){ - String key = property.getName().toLowerCase(); + String key = property.getName().toLowerCase(Locale.ROOT); AspectPropertyValue wktValue = new AspectPropertyValueDefault<>(); if(ipiObj.isNull(key)){ wktValue.setNoValueMessage(getNoValueReason(ipiObj, key)); @@ -579,7 +581,7 @@ private AspectPropertyValue getWktStringAspectPropertyValue( private AspectPropertyValue getStringAspectPropertyValue( JSONObject ipiObj, AspectPropertyMetaData property){ - String key = property.getName().toLowerCase(); + String key = property.getName().toLowerCase(Locale.ROOT); AspectPropertyValue stringValue = new AspectPropertyValueDefault<>(); if(ipiObj.isNull(key)){ stringValue.setNoValueMessage(getNoValueReason(ipiObj, key)); @@ -590,6 +592,12 @@ private AspectPropertyValue getStringAspectPropertyValue( return stringValue; } + /** + * The message for a property with no value when the response gives no + * reason. + */ + private static final String UNKNOWN_NULL_REASON = "Unknown"; + /** * Get the reason for the value of a property not being present in the cloud * engine's JSON response. @@ -599,7 +607,9 @@ private AspectPropertyValue getStringAspectPropertyValue( */ private String getNoValueReason(JSONObject ipiObj, String key){ Object reason = tryToGet(ipiObj, key + "nullreason"); - return reason == null ? null : reason.toString(); + return reason == null || reason == JSONObject.NULL ? + UNKNOWN_NULL_REASON : + reason.toString(); } /** diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/TimedHttpClient.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/TimedHttpClient.java new file mode 100644 index 000000000..7a68e173f --- /dev/null +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/TimedHttpClient.java @@ -0,0 +1,81 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.cloud; + +import fiftyone.pipeline.engines.services.HttpClientDefault; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.util.Map; + +/** + * Sends requests as the default client does, and prints how long each one + * to the live cloud service took, so each run records the measurement the + * tests' cloud timeout rests on. Only the path is printed, never the query, + * and a path's last part ending in ".json" is replaced, as it is the + * Resource Key. + */ +class TimedHttpClient extends HttpClientDefault { + + @Override + public String postData( + HttpURLConnection connection, + Map headers, + byte[] data) throws IOException { + long start = System.currentTimeMillis(); + String outcome = "failed"; + try { + String reply = super.postData(connection, headers, data); + outcome = "answered"; + return reply; + } finally { + print(connection, outcome, start); + } + } + + @Override + public String getResponseString( + HttpURLConnection connection, + Map headers) throws IOException { + long start = System.currentTimeMillis(); + String outcome = "failed"; + try { + String reply = super.getResponseString(connection, headers); + outcome = "answered"; + return reply; + } finally { + print(connection, outcome, start); + } + } + + private static void print( + HttpURLConnection connection, + String outcome, + long start) { + System.out.println( + "Cloud request " + + connection.getURL().getPath() + .replaceAll("[^/]+\\.json$", ".json") + " " + + outcome + " in " + (System.currentTimeMillis() - start) + " ms"); + } +} diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/WrapperCloud.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/WrapperCloud.java index d4c68b57a..bb75955d1 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/WrapperCloud.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/WrapperCloud.java @@ -46,8 +46,19 @@ public class WrapperCloud implements Closeable { public WrapperCloud() throws Exception { cloudRequestEngine = - new CloudRequestEngineBuilder(loggerFactory) - .setResourceKey(KeyUtils.getResourceKey("TestResourceKey")) + new CloudRequestEngineBuilder(loggerFactory, new TimedHttpClient()) + .setResourceKey(KeyUtils.getResourceKey()) + // The live cloud service is shared with every CI run. In run + // 44881631156 its slowest answer was 0.77 s, over 540 + // requests on all 15 Build and Test jobs (TimedHttpClient + // prints each one). In java #190's release build (run + // 44702767584, about 08:38 UTC on 20 November 2026) device + // detection's ValueCloudTests got "Read timed out" past the + // 2 second default on two jobs. How long those answers took + // was not measured. 15 seconds is about 20 times the slowest + // answer measured. These tests check what the service + // answers, not how fast. + .setTimeOutSeconds(15) .build(); IPIntelligenceCloudEngine = new IPIntelligenceCloudEngineBuilder(loggerFactory) .build(); diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/CloudRequestFailedTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/CloudRequestFailedTests.java new file mode 100644 index 000000000..3af3407df --- /dev/null +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/CloudRequestFailedTests.java @@ -0,0 +1,168 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.cloud.flowelements; + +import fiftyone.ipintelligence.cloud.data.IPIntelligenceDataCloud; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * When the Cloud Request Engine's request for the Flow Data failed, a + * property of the IP Intelligence cloud engine is missing with the + * CloudRequestFailed reason, rather than a reason about the resource key. + */ +public class CloudRequestFailedTests { + + private ILoggerFactory loggerFactory; + + private CloudRequestEngine requestEngine; + + @Before + public void init() { + Logger logger = mock(Logger.class); + loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + } + + @Test + public void RequestFailedWithNoResponse_CloudRequestFailed() + throws Exception { + assertCloudRequestFailed(null); + } + + @Test + public void RequestFailedWithAnErrorResponse_CloudRequestFailed() + throws Exception { + assertCloudRequestFailed("{ \"errors\": [ \"an error\" ] }"); + } + + private void assertCloudRequestFailed(String response) throws Exception { + try (IPIntelligenceCloudEngine engine = buildEngine()) { + IPIntelligenceDataCloud data = process(engine, response); + try { + data.getRegisteredName(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + assertEquals( + e.getMessage(), + "CloudRequestFailed", + e.getReason().name()); + assertEquals( + "Property 'registeredname' not found in data for " + + "element '" + engine.getElementDataKey() + "'. " + + "This is because an upstream cloud request failed, " + + "so no data was available to populate the property.", + e.getMessage()); + } + } + } + + /** + * An engine whose Cloud Request Engine reports the IP Intelligence + * product, holding a property other than the one read. + */ + // thenReturn has a varargs overload, so a generic Map is an unchecked array. + @SuppressWarnings("unchecked") + private IPIntelligenceCloudEngine buildEngine() throws Exception { + IPIntelligenceCloudEngine engine = + new IPIntelligenceCloudEngineBuilder(loggerFactory).build(); + + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("registeredowner", "String", "cat", null)); + Map publicProperties = new HashMap<>(); + publicProperties.put(engine.getElementDataKey(), product); + + requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(publicProperties); + + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + engine.addPipeline(pipeline); + return engine; + } + + private IPIntelligenceDataCloud process( + IPIntelligenceCloudEngine engine, + String response) { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn(response); + when(cloudData.getProcessStarted()).thenReturn(true); + + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + when(flowData.getErrors()).thenReturn( + Collections.singletonList(new FlowError.Default( + new Exception("The cloud request failed."), + requestEngine))); + + IPIntelligenceDataCloud data = new IPIntelligenceDataCloudInternal( + loggerFactory.getLogger( + IPIntelligenceDataCloud.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + when(flowData.getOrAdd( + ArgumentMatchers.>any(), + ArgumentMatchers.>any())) + .thenReturn(data); + try { + engine.process(flowData); + } catch (Exception e) { + // A failed request's response may not be readable. The + // property is read afterwards either way, as it is through a + // Pipeline that suppresses process exceptions. + } + return data; + } +} diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/DataModelTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/DataModelTests.java new file mode 100644 index 000000000..7bfc29912 --- /dev/null +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/DataModelTests.java @@ -0,0 +1,100 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.cloud.flowelements; + +import fiftyone.ipintelligence.cloud.data.IPIntelligenceDataCloud; +import fiftyone.ipintelligence.shared.IPIntelligenceData; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The cloud IP Intelligence engine populates Aspect Data, which is the IP + * Intelligence Element Data type. The engine processes a fixed response, + * so the test needs no resource key. + */ +public class DataModelTests { + + @Test + @SuppressWarnings("unchecked") + public void Engine_PopulatesAspectData() throws Exception { + Logger logger = mock(Logger.class); + ILoggerFactory loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + + try (IPIntelligenceCloudEngine engine = + new IPIntelligenceCloudEngineBuilder(loggerFactory).build()) { + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("registeredname", "String", "cat", null)); + Map products = new HashMap<>(); + products.put(engine.getElementDataKey(), product); + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(products); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + engine.addPipeline(pipeline); + + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn( + "{ \"ip\": { \"registeredname\": \"Example\" } }"); + when(cloudData.getProcessStarted()).thenReturn(true); + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + + IPIntelligenceDataCloud data = new IPIntelligenceDataCloudInternal( + logger, + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + when(flowData.getOrAdd(any(TypedKey.class), any())) + .thenReturn(data); + engine.process(flowData); + + assertTrue(data instanceof AspectData); + assertTrue(data instanceof IPIntelligenceData); + } + } +} diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ElementDataKeyFormatTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ElementDataKeyFormatTests.java new file mode 100644 index 000000000..d2dd2ace5 --- /dev/null +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ElementDataKeyFormatTests.java @@ -0,0 +1,69 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.cloud.flowelements; + +import org.junit.Test; +import org.slf4j.Logger; + +import java.util.Locale; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * The string key of each IP Intelligence cloud engine's data contains only + * alphanumeric characters, full stop or hyphen. + *

    + * Specification: pipeline-specification/features/access-to-results.md line 7 + * at 51Degrees/specifications 706de1f0, "Strings SHOULD only contain + * alphanumeric characters, full stop or hyphen." + */ +public class ElementDataKeyFormatTests { + + private static String[] keys() { + return new String[] { + new IPIntelligenceCloudEngine(mock(Logger.class), null) + .getElementDataKey(), + new HardwareProfileCloudEngine(mock(Logger.class), null) + .getElementDataKey() + }; + } + + @Test + public void keys_OnlyAlphanumericFullStopOrHyphen() { + for (String key : keys()) { + assertTrue(key, key.matches("[A-Za-z0-9.-]+")); + } + } + + /** + * Specification: access-to-results.md line 10, "Lower-case SHOULD be + * used." + */ + @Test + public void keys_LowerCase() { + for (String key : keys()) { + assertTrue(key, key.equals(key.toLowerCase(Locale.ROOT))); + } + } +} diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngineTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngineTests.java index 697af930d..faa9e6438 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngineTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngineTests.java @@ -23,7 +23,6 @@ package fiftyone.ipintelligence.cloud.flowelements; import fiftyone.ipintelligence.cloud.data.IPIntelligenceDataCloud; -import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; @@ -46,7 +45,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -205,19 +203,13 @@ private IPIntelligenceCloudEngine buildEngine(PropertyMetaData... properties) } /** - * Run {@code processEngine} against a mocked cloud JSON response and return + * Run {@code processCloudEngine} against a cloud JSON response and return * the populated data instance. */ private IPIntelligenceDataCloud process( IPIntelligenceCloudEngine engine, String json) { - CloudRequestData cloudData = mock(CloudRequestData.class); - when(cloudData.getJsonResponse()).thenReturn(json); - FlowData flowData = mock(FlowData.class); - when(flowData.getFromElement(any(CloudRequestEngine.class))) - .thenReturn(cloudData); - IPIntelligenceDataCloud data = new IPIntelligenceDataCloudInternal( loggerFactory.getLogger( IPIntelligenceDataCloud.class.getSimpleName()), @@ -225,7 +217,7 @@ private IPIntelligenceDataCloud process( engine, mock(MissingPropertyService.class)); - engine.processEngine(flowData, data); + engine.processCloudEngine(flowData, data, json); return data; } } diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/MissingPropertyHandlingTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/MissingPropertyHandlingTests.java index e212f527f..c2f853c1b 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/MissingPropertyHandlingTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/MissingPropertyHandlingTests.java @@ -23,7 +23,6 @@ package fiftyone.ipintelligence.cloud.flowelements; import fiftyone.ipintelligence.cloud.data.IPIntelligenceDataCloud; -import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.engines.data.AspectPropertyMetaData; @@ -63,7 +62,6 @@ public void PropertyInResource_NullValue() throws Exception { .build()) { FlowData flowData = mock(FlowData.class); - addResponse(flowData, nullValueJson); IPIntelligenceDataCloud ipiData = new IPIntelligenceDataCloudInternal( loggerFactory.getLogger(IPIntelligenceDataCloud.class.getSimpleName()), @@ -73,7 +71,7 @@ public void PropertyInResource_NullValue() throws Exception { engine.cloudRequestEngine = mock(CloudRequestEngine.class); engine.aspectProperties = properties; - engine.processEngine(flowData, ipiData); + engine.processCloudEngine(flowData, ipiData, nullValueJson); AspectPropertyValue latitude = ipiData.getLatitude(); assertNotNull(latitude); @@ -98,7 +96,6 @@ public void PropertyNotInResource() throws Exception { .build()) { FlowData flowData = mock(FlowData.class); - addResponse(flowData, nullValueJson); MissingPropertyService missingPropertyService = mock(MissingPropertyService.class); when(missingPropertyService.getMissingPropertyReason( @@ -116,7 +113,7 @@ public void PropertyNotInResource() throws Exception { engine.cloudRequestEngine = mock(CloudRequestEngine.class); engine.aspectProperties = properties; - engine.processEngine(flowData, ipiData); + engine.processCloudEngine(flowData, ipiData, nullValueJson); try { AspectPropertyValue longitude = ipiData.getLongitude(); @@ -155,11 +152,4 @@ public void PropertyNotInResource() throws Exception { add(new AspectPropertyMetaDataDefault("registeredcountry", null, null, String.class, null, true)); } }; - - private void addResponse(FlowData data, String json) { - CloudRequestData cloudData = mock(CloudRequestData.class); - when(cloudData.getJsonResponse()).thenReturn(json); - when(data.getFromElement(any(CloudRequestEngine.class))) - .thenReturn(cloudData); - } } diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java index a597f4662..d07388fd9 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java @@ -28,7 +28,9 @@ import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.typed.TypedKey; import fiftyone.pipeline.engines.exceptions.PropertyMissingException; import fiftyone.pipeline.engines.services.MissingPropertyReason; import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; @@ -97,21 +99,88 @@ public void ProductMissing_ProcessLogsAndThrowsNothing() throws Exception { } } + /** + * A failed cloud request leaves no response. The engine processes + * nothing, throws nothing and logs one warning, as the Cloud Request + * Engine's error says why. The shared Cloud Aspect Engine processing + * makes that check, so the test processes through it. + */ + @Test + public void NoResponse_ProcessesNothing() throws Exception { + try (IPIntelligenceCloudEngine engine = buildEngine("ip", "registeredname")) { + for (String response : new String[] { null, "" }) { + IPIntelligenceDataCloud data = + processFlowData(engine, response); + assertFalse(data.asKeyMap().containsKey("registeredname")); + } + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 2, logged.size()); + for (String line : logged) { + assertTrue(line, line.startsWith("warn ")); + } + } + } + /** * With the IP Intelligence product included, a response that holds no - * part for it leaves the element data without values and throws - * nothing, as the Processing section of the cloud aspect engine - * specification says. + * part for it raises nothing while processing, and reading a property + * throws PropertyMissingException, as for any property the response + * leaves out. */ @Test - public void ProductPresent_ResponseWithoutPart_NoValues() + public void ProductPresent_ResponseWithoutPart_PropertyIsMissing() throws Exception { - try (IPIntelligenceCloudEngine engine = buildEngine("ip", "registeredname")) { + try (IPIntelligenceCloudEngine engine = + buildEngine("ip", "registeredname")) { IPIntelligenceDataCloud data = process( engine, "{ 'device': { 'hardwarevendor': 'Example' }, " + "'javascriptProperties': [] }"); + try { + data.getRegisteredName(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + // The response held nothing for the product. + } + } + } + + /** + * A property the response's part leaves out has no entry, so reading it + * throws PropertyMissingException rather than giving no value. + */ + @Test + public void ProductPresent_PropertyLeftOut_PropertyIsMissing() + throws Exception { + try (IPIntelligenceCloudEngine engine = + buildEngine("ip", "registeredname")) { + IPIntelligenceDataCloud data = process( + engine, + "{ 'ip': { 'another': 'Example' }, " + + "'javascriptProperties': [] }"); + try { + data.getRegisteredName(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + // The response's part did not hold the property. + } + } + } + + /** + * A null with no reason beside it is no value with the message Unknown. + */ + @Test + public void NullWithoutReason_MessageIsUnknown() throws Exception { + try (IPIntelligenceCloudEngine engine = + buildEngine("ip", "registeredname")) { + IPIntelligenceDataCloud data = process( + engine, + "{ 'ip': { 'registeredname': null }, " + + "'javascriptProperties': [] }"); assertFalse(data.getRegisteredName().hasValue()); + assertEquals("Unknown", + data.getRegisteredName().getNoValueMessage()); } } @@ -198,25 +267,48 @@ private IPIntelligenceCloudEngine buildEngine( } /** - * Process the given cloud response. + * Process a Flow Data whose Cloud Request Engine ran and gave the + * response, as a Pipeline does, through the engine's public process + * method. */ - private IPIntelligenceDataCloud process( + @SuppressWarnings("unchecked") + private IPIntelligenceDataCloud processFlowData( IPIntelligenceCloudEngine engine, - String response) { + String response) throws Exception { CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getProcessStarted()).thenReturn(true); when(cloudData.getJsonResponse()).thenReturn(response); FlowData flowData = mock(FlowData.class); when(flowData.getFromElement(any(CloudRequestEngine.class))) .thenReturn(cloudData); + IPIntelligenceDataCloud data = new IPIntelligenceDataCloudInternal( + loggerFactory.getLogger( + IPIntelligenceDataCloud.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + when(flowData.getOrAdd( + any(TypedKey.class), + any(FlowElement.DataFactory.class))).thenReturn(data); + engine.process(flowData); + return data; + } + /** + * Process the given cloud response. + */ + private IPIntelligenceDataCloud process( + IPIntelligenceCloudEngine engine, + String response) { + FlowData flowData = mock(FlowData.class); IPIntelligenceDataCloud data = new IPIntelligenceDataCloudInternal( loggerFactory.getLogger( IPIntelligenceDataCloud.class.getSimpleName()), flowData, engine, MissingPropertyServiceDefault.getInstance()); - engine.processEngine(flowData, data); + engine.processCloudEngine(flowData, data, response); return data; } diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/SharedCloudProcessingTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/SharedCloudProcessingTests.java new file mode 100644 index 000000000..8f7ed6af5 --- /dev/null +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/SharedCloudProcessingTests.java @@ -0,0 +1,154 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.cloud.flowelements; + +import fiftyone.ipintelligence.cloud.data.IPIntelligenceDataCloud; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The cloud IP Intelligence engine leaves its processing to the shared Cloud + * Aspect Engine logic, which finds the Cloud Request Engine's data, checks + * the request ran, and passes on only a response that is there, so the + * engine itself only reads its part of the response. The engine is driven + * through its public process method with a fixed response, so the tests + * need no resource key. + */ +public class SharedCloudProcessingTests { + + private final Logger logger = mock(Logger.class); + + private final CloudRequestData cloudData = mock(CloudRequestData.class); + + private final FlowData flowData = mock(FlowData.class); + + private IPIntelligenceCloudEngine engine; + + private IPIntelligenceDataCloud data; + + @Before + @SuppressWarnings("unchecked") + public void init() throws Exception { + ILoggerFactory loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + engine = new IPIntelligenceCloudEngineBuilder(loggerFactory).build(); + + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("registeredname", "String", "cat", null)); + Map products = new HashMap<>(); + products.put(engine.getElementDataKey(), product); + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(products); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + engine.addPipeline(pipeline); + + data = new IPIntelligenceDataCloudInternal( + logger, + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + when(flowData.getOrAdd(any(TypedKey.class), any())) + .thenReturn(data); + } + + @After + public void cleanup() throws Exception { + engine.close(); + } + + @Test + public void Engine_DeclaresOnlyTheResponseReading() throws Exception { + for (Method method : + IPIntelligenceCloudEngine.class.getDeclaredMethods()) { + if (method.getName().equals("processEngine") && + method.isSynthetic() == false) { + fail("The engine should leave processEngine to the shared " + + "Cloud Aspect Engine logic."); + } + } + IPIntelligenceCloudEngine.class.getDeclaredMethod( + "processCloudEngine", + FlowData.class, + IPIntelligenceDataCloud.class, + String.class); + } + + @Test + public void Response_Read() throws Exception { + when(cloudData.getProcessStarted()).thenReturn(true); + when(cloudData.getJsonResponse()).thenReturn( + "{ \"ip\": { \"registeredname\": \"Example\" } }"); + engine.process(flowData); + assertEquals("Example", data.getRegisteredName().getValue()); + } + + @Test + public void NoResponse_WarnsAndStops() throws Exception { + when(cloudData.getProcessStarted()).thenReturn(true); + when(cloudData.getJsonResponse()).thenReturn(null); + engine.process(flowData); + verify(logger).warn(contains("did not process")); + assertTrue(data.asKeyMap().isEmpty()); + } + + @Test(expected = PipelineConfigurationException.class) + public void RequestNotRun_Throws() throws Exception { + when(cloudData.getProcessStarted()).thenReturn(false); + when(cloudData.getJsonResponse()).thenReturn( + "{ \"ip\": { \"registeredname\": \"Example\" } }"); + engine.process(flowData); + } +} diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/WritesAfterProcessingTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/WritesAfterProcessingTests.java new file mode 100644 index 000000000..02c08ea25 --- /dev/null +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/WritesAfterProcessingTests.java @@ -0,0 +1,186 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.cloud.flowelements; + +import fiftyone.ipintelligence.cloud.data.IPIntelligenceDataCloud; +import fiftyone.ipintelligence.cloud.data.MultiIPIDataCloud; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudAspectEngineBase; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.TryGetResult; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.FlowElement.DataFactory; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.engines.data.AspectDataBase; +import org.junit.Test; +import org.junit.function.ThrowingRunnable; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The check, for the Flow Elements in this module, that processes each one, + * then writes to its Element Data and expects the error: the + * {@link IPIntelligenceCloudEngine} and the + * {@link HardwareProfileCloudEngine}, whose profiles are written only while + * it processes as well. Each is processed through its public process method + * with a fixed cloud response, so the check needs no resource key. + * @see Specification + */ +public class WritesAfterProcessingTests { + + private static final ILoggerFactory LOGGER_FACTORY = + LoggerFactory.getILoggerFactory(); + + private static void assertFails(String what, ThrowingRunnable write) { + assertThrows(what, IllegalStateException.class, write); + } + + /** + * Every write method of Element Data, and of Aspect Data, fails once its + * Flow Element has finished. + * @param data the Element Data of a finished Flow Element + */ + static void assertWritesFail(ElementData data) { + assertFails("put", () -> data.put("after", "value")); + assertFails("populateFromMap", () -> data.populateFromMap( + Collections.singletonMap("after", "value"))); + if (data instanceof AspectDataBase) { + AspectDataBase aspect = (AspectDataBase) data; + assertFails("setCloudRequestFailed", aspect::setCloudRequestFailed); + assertFails("addEngine", () -> aspect.addEngine(null)); + assertFails("addProcessCallable", + () -> aspect.addProcessCallable(null)); + } + } + + /** + * Attach the engine to a Pipeline whose Cloud Request Engine reports the + * product given. + */ + private static > T attach( + T engine, + String productKey, + PropertyMetaData property) { + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Collections.singletonList(property); + Map products = new HashMap<>(); + products.put(productKey, product); + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(products); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + engine.addPipeline(pipeline); + return engine; + } + + /** + * Process the engine through its public process method, with a Flow + * Data stand-in that holds the Element Data the engine adds and gives + * the Cloud Request Engine's data with the response given. + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + private static T process( + FlowElement engine, + String response) throws Exception { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getProcessStarted()).thenReturn(true); + when(cloudData.getJsonResponse()).thenReturn(response); + final FlowData flowData = mock(FlowData.class); + final List added = new ArrayList<>(); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + when(flowData.getOrAdd(any(TypedKey.class), any(DataFactory.class))) + .thenAnswer(invocation -> { + if (added.isEmpty()) { + DataFactory factory = invocation.getArgument(1); + added.add(factory.create(flowData)); + } + return added.get(0); + }); + when(flowData.tryGetValue(any(TypedKey.class))) + .thenAnswer(invocation -> { + TryGetResult result = new TryGetResult<>(); + if (added.isEmpty() == false) { + result.setValue(added.get(0)); + } + return result; + }); + engine.process(flowData); + assertEquals(1, added.size()); + return (T) added.get(0); + } + + @Test + public void ipIntelligenceCloudEngine_ElementDataRefusesWrites() + throws Exception { + try (IPIntelligenceCloudEngine engine = attach( + new IPIntelligenceCloudEngineBuilder(LOGGER_FACTORY).build(), + "ip", + new PropertyMetaData("registeredname", "String", "cat", null))) { + IPIntelligenceDataCloud own = process( + engine, + "{ \"ip\": { \"registeredname\": \"Example\" } }"); + assertEquals("Example", own.getRegisteredName().getValue()); + assertWritesFail(own); + } + } + + @Test + public void hardwareProfileCloudEngine_ElementDataRefusesWrites() + throws Exception { + try (HardwareProfileCloudEngine engine = attach( + new HardwareProfileCloudEngineBuilder(LOGGER_FACTORY).build(), + "hardware", + new PropertyMetaData("profiles", "Array", "cat", + Collections.singletonList(new PropertyMetaData( + "hardwarevendor", "String", "cat", null))))) { + MultiIPIDataCloud own = process( + engine, + "{ \"hardware\": { \"profiles\": [ " + + "{ \"hardwarevendor\": \"Example\" } ] } }"); + assertEquals(1, own.getProfiles().size()); + assertWritesFail(own); + assertFails("addProfile", () -> own.addProfile(null)); + assertWritesFail(own.getProfiles().get(0)); + } + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/pom.xml b/ip-intelligence/ip-intelligence.engine.on-premise/pom.xml index 0c48477de..da9985ff0 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/pom.xml +++ b/ip-intelligence/ip-intelligence.engine.on-premise/pom.xml @@ -11,7 +11,7 @@ ip-intelligence.engine.on-premise 51Degrees :: IP Intelligence :: On Premise - Provides the On-premise IP Intelligence engine for the 51Degrees Pipeline API. This can be used as an alternative to legacy IP Intelligence services such as MaxMind or IPinfo. + On-premise IP Intelligence and IP geolocation engine for Java. Reads a 51Degrees IP Intelligence data file on your server to find an IP address's likely location (country, region, town, and coordinates), connection type, and network. Built on the 51Degrees Pipeline API. An alternative to MaxMind and IPinfo. https://github.com/51Degrees/51degrees-java diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPI.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPI.java index eabb2aecb..bcd58b4de 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPI.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPI.java @@ -206,6 +206,11 @@ public Class getType() { return Boolean.class; case "double": return Double.class; + // A 32 bit floating point number, as Latitude and Longitude + // are, which the data model types as float. + case "single": + case "weightedsingle": + return Float.class; case "javascript": return JavaScript.class; case "string[]": diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java index 6d1300fd8..b2b6cfcf0 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java @@ -74,6 +74,12 @@ public class IPIntelligenceDataHashDefault */ private final IPIntelligenceOnPremiseEngine ipiEngine; + /** + * Held for every read from {@link #resultsList}, so that only one thread + * at a time reads from the native results. + */ + private final Object resultsLock = new Object(); + /** * Constructs a new instance. * @param logger used for logging @@ -142,6 +148,11 @@ private ResultsIpiSwig getResultsContainingProperty(String propertyName) { @Override protected boolean propertyIsAvailable(String propertyName) { checkState(); + // A property the engine has a position for is held by its results, + // so that needs no lookup of the name in native code. + if (indexOf(propertyName) != null) { + return resultsList.isEmpty() == false; + } for (ResultsIpiSwig results : resultsList) { if (results.containsProperty(propertyName)) { return true; @@ -184,6 +195,13 @@ void populateWeightedValues( public AspectPropertyValue> getValues(String propertyName, Class[] parametrizedTypes) { checkState(); + synchronized (resultsLock) { + return readValues(propertyName, parametrizedTypes); + } + } + + private AspectPropertyValue> readValues(String propertyName, + Class[] parametrizedTypes) { AspectPropertyValue> result = new AspectPropertyValueDefault<>(); ResultsIpiSwig results = getSingleResults(propertyName); @@ -396,13 +414,19 @@ protected AspectPropertyValue getValueAsDouble(String propertyName) { return result; } + /** + * Holds the results lock for the whole read, as finding whether a + * property is available reads the native results too. + */ @Override protected TryGetResult tryGetValue( String key, Class type, Class... parameterisedTypes) { checkState(); - return super.tryGetValue(key, type, parameterisedTypes); + synchronized (resultsLock) { + return super.tryGetValue(key, type, parameterisedTypes); + } } private void checkState() { diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngine.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngine.java index 6955ef9e9..7327bb6f0 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngine.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngine.java @@ -53,7 +53,14 @@ public class IPIntelligenceOnPremiseEngine extends FiftyOneOnPremiseAspectEngineBase { - private EngineIpiSwig engine = null; + // Volatile, so a thread that processes sees the native engine a + // refresh on another thread made. + private volatile EngineIpiSwig engine = null; + // Set, while this engine's monitor is held, once closing has freed the + // native objects, so a refresh that gets the monitor afterwards fails + // instead of using them. It has no initializer, so an engine built by + // a mock that skips field initializers still has it, as false. + private boolean freed; private final List properties = new ArrayList<>(); private final ConfigIpiSwig config; private final RequiredPropertiesConfigSwig propertiesConfigSwig; @@ -324,8 +331,42 @@ public EvidenceKeyFilter getEvidenceKeyFilter() { return evidenceKeyFilter; } + /** + * {@inheritDoc} + * Only one refresh of this engine runs at a time, from either overload, + * whichever route calls it. A second refresh waits for the first to + * finish. Processing takes no part in this and never waits for a + * refresh. + *

    + * The lock is this engine's own monitor, held for the whole refresh. + * Closing the engine takes it too, so the native objects are freed + * only once a refresh that is running has finished. Nothing else in + * the engine or the Pipeline takes it, and no other lock is taken + * while it is held, so it cannot deadlock. It is not a + * field, so an engine built by a mock that skips field initializers + * still has it. + */ @Override public void refreshData(String dataFileIdentifier) { + synchronized (this) { + refuseIfFreed(); + refreshFromFile(); + } + } + + /** + * Called with this engine's monitor held, before a refresh uses the + * native objects. + * @throws IllegalStateException if the engine has been closed + */ + private void refuseIfFreed() { + if (freed) { + throw new IllegalStateException( + "The engine has been closed, so its data cannot be refreshed."); + } + } + + private void refreshFromFile() { AspectEngineDataFile dataFile = getDataFiles().get(0); if (Objects.isNull(engine)) { @@ -357,14 +398,22 @@ public void refreshData(String dataFileIdentifier) { setEngineMetaData(); } + /** + * {@inheritDoc} + * Only one refresh of this engine runs at a time, as + * {@link #refreshData(String)} describes. + */ @Override public void refreshData(String dataFileIdentifier, byte[] data) { - if (engine == null) { - engine = new EngineIpiSwig(data, config, propertiesConfigSwig); - } else { - engine.refreshData(data); + synchronized (this) { + refuseIfFreed(); + if (engine == null) { + engine = new EngineIpiSwig(data, config, propertiesConfigSwig); + } else { + engine.refreshData(data); + } + setEngineMetaData(); } - setEngineMetaData(); } @Override @@ -402,14 +451,23 @@ protected void unmanagedResourcesCleanup() { } } } - if (config != null) { - config.delete(); - } - if (propertiesConfigSwig != null) { - propertiesConfigSwig.delete(); - } - if (engine != null) { - engine.delete(); + // A refresh holds this engine's monitor while it uses the native + // objects, so taking it here waits for a refresh that is running + // before they are freed. + synchronized (this) { + if (freed) { + return; + } + freed = true; + if (config != null) { + config.delete(); + } + if (propertiesConfigSwig != null) { + propertiesConfigSwig.delete(); + } + if (engine != null) { + engine.delete(); + } } } diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java index cf0b071fe..cd7a9010e 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java @@ -159,6 +159,7 @@ public IPIntelligenceOnPremiseEngineBuilder setPerformanceProfile( } } + @DefaultValue("Balanced") @Override public IPIntelligenceOnPremiseEngineBuilder setPerformanceProfile( PerformanceProfiles profile) { @@ -201,6 +202,7 @@ public IPIntelligenceOnPremiseEngineBuilder setConcurrency(int concurrency) { return this; } + @DefaultValue("No results cache, and none can be set for this engine") @Override public IPIntelligenceOnPremiseEngineBuilder setCache( CacheConfiguration cacheConfiguration) { diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPITypeTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPITypeTests.java new file mode 100644 index 000000000..cf897474d --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/data/PropertyMetaDataIPITypeTests.java @@ -0,0 +1,78 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.data; + +import fiftyone.ipintelligence.engine.onpremise.interop.swig.PropertyMetaDataSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.VectorStringSwig; +import fiftyone.pipeline.core.data.types.JavaScript; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The Java type a property's meta data gives for each type name the native + * meta data uses. The element data asks for a property's values in this + * type when the caller names no type, as get(name) does. + * + * The IP Intelligence data model gives Latitude and Longitude the type + * float (ip-intelligence-specification/data-model.md at + * 51Degrees/specifications 706de1f0), and their typed accessors return + * Float. The native meta data names them "single", a 32 bit floating point + * number, or "weightedsingle" for the weighted form. + */ +public class PropertyMetaDataIPITypeTests { + + private static Class typeOf(String nativeType) { + PropertyMetaDataSwig source = mock(PropertyMetaDataSwig.class); + when(source.getType()).thenReturn(nativeType); + when(source.getName()).thenReturn("Property"); + when(source.getDataFilesWherePresent()) + .thenReturn(mock(VectorStringSwig.class)); + return new PropertyMetaDataIPI(null, source).getType(); + } + + @Test + public void Single_IsFloat() { + assertEquals(Float.class, typeOf("single")); + } + + @Test + public void WeightedSingle_IsFloat() { + assertEquals(Float.class, typeOf("weightedsingle")); + } + + @Test + public void OtherNames_AreUnchanged() { + assertEquals(String.class, typeOf("string")); + assertEquals(Integer.class, typeOf("int")); + assertEquals(Boolean.class, typeOf("bool")); + assertEquals(Double.class, typeOf("double")); + assertEquals(JavaScript.class, typeOf("javascript")); + assertEquals(List.class, typeOf("string[]")); + assertEquals(String.class, typeOf("not a type name")); + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/AcceptedEvidenceRefreshTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/AcceptedEvidenceRefreshTests.java new file mode 100644 index 000000000..5aad4686c --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/AcceptedEvidenceRefreshTests.java @@ -0,0 +1,112 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.TestsBase; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EngineIpiSwig; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.engines.Constants; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * The engine reads the Evidence keys the data file accepts from the native + * engine when it starts and again whenever the data is refreshed. Both free + * data files accept the same keys, so the test puts a key no data file has + * in place of the keys before a refresh, and checks that the refresh + * replaces it with the keys the native engine gives. + */ +public class AcceptedEvidenceRefreshTests extends TestsBase { + + private static final String STALE_KEY = "query.stale-key"; + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.LowMemory); + } + + @After + public void cleanup() { + testCleanup(); + } + + @Test + public void StartUp_KeysAreTheNativeEngines() throws Exception { + IPIntelligenceOnPremiseEngine engine = getWrapper().getEngine(); + assertEquals(nativeKeys(engine), filterKeys(engine)); + } + + @Test + public void Refresh_ReadsTheKeysAgain() throws Exception { + IPIntelligenceOnPremiseEngine engine = getWrapper().getEngine(); + List stale = Collections.singletonList(STALE_KEY); + set(engine, "evidenceKeys", stale); + set(engine, "evidenceKeyFilter", new EvidenceKeyFilterWhitelist( + stale, String.CASE_INSENSITIVE_ORDER)); + + engine.refreshData(engine.getDataFileMetaData().getIdentifier()); + + TreeSet keys = filterKeys(engine); + assertFalse(keys.toString(), keys.contains(STALE_KEY)); + assertEquals(nativeKeys(engine), keys); + } + + private static TreeSet filterKeys( + IPIntelligenceOnPremiseEngine engine) { + Map whitelist = ((EvidenceKeyFilterWhitelist) + engine.getEvidenceKeyFilter()).getWhitelist(); + return new TreeSet<>(whitelist.keySet()); + } + + private static TreeSet nativeKeys( + IPIntelligenceOnPremiseEngine engine) throws Exception { + EngineIpiSwig nativeEngine = (EngineIpiSwig) get(engine, "engine"); + return new TreeSet<>(new ArrayList<>(nativeEngine.getKeys())); + } + + private static Object get(Object target, String name) throws Exception { + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } + + private static void set(Object target, String name, Object value) + throws Exception { + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/AngleRoundingTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/AngleRoundingTests.java new file mode 100644 index 000000000..3d26cf1ae --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/AngleRoundingTests.java @@ -0,0 +1,188 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.shared.IPIntelligenceData; +import fiftyone.ipintelligence.shared.IPIntelligenceDataBaseOnPremise; +import fiftyone.ipintelligence.shared.testhelpers.FileUtils; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.IWeightedValue; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; + +/** + * Latitude and Longitude are stored as Declination and Azimuth, so every + * form they are given in is rounded to 3 decimal places. See + * value-types.md line 99: + * "A value converted from Azimuth or Declination MUST be given to the + * caller rounded to 3 decimal places, whether it is given as a number, as a + * string or as a point in a WKT shape." + *

    + * The free Lite file has no Latitude or Longitude, so these tests need a + * paid data file and are skipped without one. + */ +public class AngleRoundingTests { + + private static final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + private static final List ANGLES = + Arrays.asList("Latitude", "Longitude"); + + /** + * Addresses in different parts of the world, so that the values are + * not all whole numbers. + */ + private static final List ADDRESSES = Arrays.asList( + "185.28.167.77", + "116.154.188.222", + "45.236.48.61", + "2001:4860:4860::8888"); + + private interface Check { + void accept(IPIntelligenceData data) throws Exception; + } + + private static void forEachAddress(Check check) throws Exception { + File file = FileUtils.getHashFile(); + assumeTrue("No IP Intelligence data file was found.", file != null); + try (IPIntelligenceOnPremiseEngine engine = + new IPIntelligenceOnPremiseEngineBuilder(loggerFactory, null) + .setPerformanceProfile( + Constants.PerformanceProfiles.MaxPerformance) + .setAutoUpdate(false) + .setDataFileSystemWatcher(false) + .build(file.toString(), false); + Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build()) { + for (String angle : ANGLES) { + assumeTrue( + "The data file has no " + angle + ".", + engine.getProperties().stream().anyMatch( + p -> p.getName().equalsIgnoreCase(angle))); + } + for (String address : ADDRESSES) { + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.client-ip", address); + flowData.process(); + check.accept(flowData.get(IPIntelligenceData.class)); + } + } + } + } + + private static void assertRounded(String angle, double value) { + double rounded = Math.round(value * 1000d) / 1000d; + assertEquals( + angle + " " + value + " is not rounded to 3 decimal places.", + rounded, + value, + 0d); + } + + private static void assertRounded(String angle, float value) { + float rounded = (float) (Math.round(value * 1000d) / 1000d); + assertEquals( + angle + " " + value + " is not rounded to 3 decimal places.", + rounded, + value, + 0f); + } + + /** + * A weighted list of doubles gives each value rounded. + */ + @Test + public void weightedDoubles_AreRounded() throws Exception { + forEachAddress(data -> { + for (String angle : ANGLES) { + for (Object value : weighted(data, angle, Double.class)) { + assertRounded(angle, (double) (Double) value); + } + } + }); + } + + /** + * The values of a weighted list, read through the public getValues. + */ + private static List weighted( + IPIntelligenceData data, String angle, Class type) { + AspectPropertyValue> values = + ((IPIntelligenceDataBaseOnPremise) data).getValues( + angle, + new Class[] { List.class, IWeightedValue.class, type }); + List result = new java.util.ArrayList<>(); + if (values != null && values.hasValue()) { + for (Object value : values.getValue()) { + result.add(((IWeightedValue) value).getValue()); + } + } + return result; + } + + /** + * A weighted list of floats gives each value rounded. + */ + @Test + public void weightedFloats_AreRounded() throws Exception { + forEachAddress(data -> { + for (String angle : ANGLES) { + for (Object value : weighted(data, angle, Float.class)) { + assertRounded(angle, (float) (Float) value); + } + } + }); + } + + /** + * The single number is rounded. The text the native results give is + * not reachable through a public accessor, so it is not checked here. + */ + @Test + public void number_IsRounded() throws Exception { + forEachAddress(data -> { + AspectPropertyValue latitude = data.getLatitude(); + if (latitude.hasValue()) { + assertRounded("Latitude", latitude.getValue()); + } + AspectPropertyValue longitude = data.getLongitude(); + if (longitude.hasValue()) { + assertRounded("Longitude", longitude.getValue()); + } + }); + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ClientIpChoiceTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ClientIpChoiceTests.java new file mode 100644 index 000000000..b8561d5c5 --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ClientIpChoiceTests.java @@ -0,0 +1,115 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.TestsBase; +import fiftyone.ipintelligence.shared.IPIntelligenceData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +/** + * When the Evidence holds more than one client IP key, one address is used, + * a query key ahead of a server key, and the Property values describe that + * address. + */ +public class ClientIpChoiceTests extends TestsBase { + + private static final String QUERY_KEY = "query.client-ip"; + + private static final String SERVER_KEY = "server.true-client-ip-51d"; + + private static final String QUERY_ADDRESS = "116.154.188.222"; + + private static final String SERVER_ADDRESS = "185.28.167.77"; + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.LowMemory); + } + + @After + public void cleanup() { + testCleanup(); + } + + @Test + public void QueryAndServerKeys_QueryAddressUsed() throws Exception { + assertAccepted(QUERY_KEY); + assertAccepted(SERVER_KEY); + Map query = values(QUERY_KEY, QUERY_ADDRESS); + Map server = values(SERVER_KEY, SERVER_ADDRESS); + assertNotEquals( + "Expected the two addresses to give different values.", + query, + server); + + assertEquals(query, values( + QUERY_KEY, QUERY_ADDRESS, SERVER_KEY, SERVER_ADDRESS)); + assertEquals(query, values( + SERVER_KEY, SERVER_ADDRESS, QUERY_KEY, QUERY_ADDRESS)); + } + + @Test + public void ServerKeyOnly_ServerAddressUsed() throws Exception { + assertAccepted(SERVER_KEY); + assertEquals( + values(QUERY_KEY, SERVER_ADDRESS), + values(SERVER_KEY, SERVER_ADDRESS)); + } + + private void assertAccepted(String key) { + assertTrue("Expected the engine to accept '" + key + "'.", + getWrapper().getEngine().getEvidenceKeyFilter().include(key)); + } + + /** + * Process Evidence given as key, value pairs and give the registered + * name, country and owner. + */ + private Map values(String... evidence) throws Exception { + Map values = new LinkedHashMap<>(); + try (FlowData data = getWrapper().getPipeline().createFlowData()) { + for (int i = 0; i < evidence.length; i += 2) { + data.addEvidence(evidence[i], evidence[i + 1]); + } + data.process(); + IPIntelligenceData ipi = data.get(IPIntelligenceData.class); + values.put("RegisteredName", ipi.getRegisteredName().getValue()); + values.put("RegisteredCountry", + ipi.getRegisteredCountry().getValue()); + values.put("RegisteredOwner", + ipi.getRegisteredOwner().getValue()); + } + return values; + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/DataModelTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/DataModelTests.java new file mode 100644 index 000000000..de236bab5 --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/DataModelTests.java @@ -0,0 +1,72 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.TestsBase; +import fiftyone.ipintelligence.shared.IPIntelligenceData; +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.data.AspectData; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** + * The on-premise IP Intelligence engine populates Aspect Data, and the IP + * Intelligence Element Data type extends Aspect Data. + */ +public class DataModelTests extends TestsBase { + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.LowMemory); + } + + @After + public void cleanup() { + testCleanup(); + } + + @Test + public void ElementDataType_ExtendsAspectData() { + assertTrue(AspectData.class.isAssignableFrom( + IPIntelligenceData.class)); + } + + @Test + public void Engine_PopulatesAspectData() throws Exception { + IPIntelligenceOnPremiseEngine engine = getWrapper().getEngine(); + try (FlowData data = getWrapper().getPipeline().createFlowData()) { + data.addEvidence("query.client-ip", "116.154.188.222"); + data.process(); + ElementData element = data.get(engine.getElementDataKey()); + assertTrue(String.valueOf(element), + element instanceof AspectData); + assertTrue(String.valueOf(element), + element instanceof IPIntelligenceData); + } + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/DataRefreshTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/DataRefreshTests.java new file mode 100644 index 000000000..95f1b6a03 --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/DataRefreshTests.java @@ -0,0 +1,174 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.TestsBase; +import fiftyone.ipintelligence.shared.IPIntelligenceData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Refreshing the engine's data from several threads at once, with the free + * Lite data file. The engine lets only one refresh run at a time, so every + * refresh completes and the engine still works afterwards. + */ +public class DataRefreshTests extends TestsBase { + + private static final int THREADS = 4; + + private static final int REFRESHES = 5; + + private static final long WAIT_SECONDS = 300; + + private static final String ADDRESS = "116.154.188.222"; + + private ExecutorService executor; + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.LowMemory); + executor = Executors.newFixedThreadPool(THREADS); + } + + @After + public void cleanup() { + executor.shutdownNow(); + testCleanup(); + } + + /** + * Several threads refresh the engine from its data file at once, and + * the engine then gives the values it gave before. + */ + @Test + public void Refresh_FromSeveralThreads_EngineStillWorks() + throws Exception { + String expected = registeredCountry(); + IPIntelligenceOnPremiseEngine engine = getWrapper().getEngine(); + String identifier = engine.getDataFileMetaData().getIdentifier(); + CountDownLatch start = new CountDownLatch(1); + List> refreshers = new ArrayList<>(); + for (int t = 0; t < THREADS; t++) { + refreshers.add(executor.submit(() -> { + start.await(); + for (int i = 0; i < REFRESHES; i++) { + engine.refreshData(identifier); + } + return REFRESHES; + })); + } + start.countDown(); + for (Future refresher : refreshers) { + assertEquals(Integer.valueOf(REFRESHES), + refresher.get(WAIT_SECONDS, TimeUnit.SECONDS)); + } + assertEquals(expected, registeredCountry()); + } + + /** + * The registered country of one address. + */ + private String registeredCountry() throws Exception { + try (FlowData data = getWrapper().getPipeline().createFlowData()) { + data.addEvidence("query.client-ip", ADDRESS); + data.process(); + return data.get(IPIntelligenceData.class) + .getRegisteredCountry().getValue(); + } + } + + /** + * Closing the engine while a refresh runs waits for the refresh to + * finish, so the native engine is never freed under it. The test holds + * the engine's own monitor, which a refresh holds while it runs. + */ + @Test + public void Close_WhileARefreshRuns_WaitsForTheRefresh() + throws Exception { + IPIntelligenceOnPremiseEngine engine = getWrapper().getEngine(); + CountDownLatch held = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + Future refresh = executor.submit(() -> { + synchronized (engine) { + held.countDown(); + return release.await(WAIT_SECONDS, TimeUnit.SECONDS); + } + }); + assertTrue(held.await(WAIT_SECONDS, TimeUnit.SECONDS)); + Future closing = executor.submit(() -> { + engine.close(); + return true; + }); + try { + closing.get(2, TimeUnit.SECONDS); + fail("The engine closed while a refresh was running."); + } catch (TimeoutException e) { + // Closing waits for the refresh, as it should. + } finally { + release.countDown(); + } + assertTrue(refresh.get(WAIT_SECONDS, TimeUnit.SECONDS)); + assertTrue(closing.get(WAIT_SECONDS, TimeUnit.SECONDS)); + } + + /** + * A refresh that gets the engine's monitor after the engine is closed, + * from either overload, fails with an error and does not use the native + * objects that closing freed. + */ + @Test + public void Refresh_AfterClose_FailsWithoutUsingTheFreedEngine() + throws Exception { + IPIntelligenceOnPremiseEngine engine = getWrapper().getEngine(); + String identifier = engine.getDataFileMetaData().getIdentifier(); + engine.close(); + try { + engine.refreshData(identifier); + fail("A refresh from file ran after the engine was closed."); + } catch (IllegalStateException e) { + // As it should. + } + try { + engine.refreshData(identifier, new byte[] { 0 }); + fail("A refresh from memory ran after the engine was closed."); + } catch (IllegalStateException e) { + // As it should. + } + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/EngineNamesTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/EngineNamesTests.java new file mode 100644 index 000000000..3c49e78b3 --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/EngineNamesTests.java @@ -0,0 +1,132 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EngineIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.RequiredPropertiesConfigSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.VectorStringSwig; +import fiftyone.pipeline.engines.Constants; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static fiftyone.ipintelligence.shared.testhelpers.FileUtils.LITE_IPI_DATA_FILE_NAME; +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; + +/** + * The names the native engine is created with, read from the configuration + * the engine passes to it, and the names the native engine then holds. + */ +public class EngineNamesTests { + + private static final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + @Test + public void NoNames_NativeEngineCreatedWithNoNames() throws Exception { + try (IPIntelligenceOnPremiseEngine engine = build(null)) { + assertEquals( + "An engine configured with no names should create the " + + "native engine with no names.", + Collections.emptyList(), + namesCreatedWith(engine)); + } + } + + @Test + public void Names_NativeEngineCreatedWithOnlyThoseNames() + throws Exception { + List names = Collections.singletonList("RegisteredName"); + try (IPIntelligenceOnPremiseEngine engine = build(names)) { + assertEquals( + "An engine configured with names should create the native " + + "engine with those names and no others.", + names, + namesCreatedWith(engine)); + assertEquals( + "The native engine should hold only the names configured.", + names, + nativeNames(engine)); + } + } + + static IPIntelligenceOnPremiseEngine build(List names) + throws Exception { + File file; + try { + file = getFilePath(LITE_IPI_DATA_FILE_NAME); + } + catch (IllegalArgumentException e) { + assumeTrue(e.getMessage(), false); + return null; + } + IPIntelligenceOnPremiseEngineBuilder builder = + new IPIntelligenceOnPremiseEngineBuilder(loggerFactory, null) + .setPerformanceProfile( + Constants.PerformanceProfiles.MaxPerformance) + .setAutoUpdate(false) + .setDataFileSystemWatcher(false); + if (names != null) { + builder.setProperties(names); + } + return builder.build(file.toString(), false); + } + + /** + * The names in the configuration the engine creates its native engine + * with, and creates it again with when the native engine is replaced. + */ + static List namesCreatedWith( + IPIntelligenceOnPremiseEngine engine) throws Exception { + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField("propertiesConfigSwig"); + field.setAccessible(true); + RequiredPropertiesConfigSwig config = + (RequiredPropertiesConfigSwig) field.get(engine); + try (VectorStringSwig names = config.getProperties()) { + return new ArrayList<>(names); + } + } + + /** + * The names the native engine reports it was created with. + */ + static List nativeNames(IPIntelligenceOnPremiseEngine engine) + throws Exception { + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField("engine"); + field.setAccessible(true); + EngineIpiSwig nativeEngine = (EngineIpiSwig) field.get(engine); + try (VectorStringSwig names = nativeEngine.getRequiredProperties()) { + return new ArrayList<>(names); + } + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ProcessMarshallingTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ProcessMarshallingTests.java new file mode 100644 index 000000000..6b099b1a7 --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ProcessMarshallingTests.java @@ -0,0 +1,133 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.TestsBase; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EngineIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EvidenceIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.ResultsIpiSwig; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Processing keeps what crosses into native code small. It passes the native + * engine only the Evidence the engine accepts, and takes the accepted keys + * from the native engine once for each data set, never on each process. A + * stand-in for the native engine records what it is given and asked. + */ +public class ProcessMarshallingTests extends TestsBase { + + private static final String CLIENT_IP_KEY = "query.client-ip"; + + private static final String ADDRESS = "116.154.188.222"; + + private static final int OTHER_EVIDENCE = 20; + + private static final int PROCESSES = 5; + + private final List> passed = new ArrayList<>(); + + private EngineIpiSwig nativeEngine; + + private Field engineField; + + private Object realEngine; + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.LowMemory); + nativeEngine = mock(EngineIpiSwig.class); + when(nativeEngine.process(any(EvidenceIpiSwig.class))) + .thenAnswer(invocation -> { + passed.add(new TreeMap<>( + (EvidenceIpiSwig) invocation.getArgument(0))); + return mock(ResultsIpiSwig.class); + }); + engineField = IPIntelligenceOnPremiseEngine.class + .getDeclaredField("engine"); + engineField.setAccessible(true); + realEngine = engineField.get(getWrapper().getEngine()); + when(nativeEngine.getKeys()) + .thenReturn(((EngineIpiSwig) realEngine).getKeys()); + engineField.set(getWrapper().getEngine(), nativeEngine); + } + + @After + public void cleanup() throws Exception { + engineField.set(getWrapper().getEngine(), realEngine); + testCleanup(); + } + + @Test + public void Process_PassesOnlyAcceptedEvidence() throws Exception { + try (FlowData data = getWrapper().getPipeline().createFlowData()) { + data.addEvidence(CLIENT_IP_KEY, ADDRESS); + for (int i = 0; i < OTHER_EVIDENCE; i++) { + data.addEvidence("query.other-" + i, "value " + i); + data.addEvidence("cookie.other-" + i, "value " + i); + } + data.process(); + } + assertEquals(1, passed.size()); + assertEquals( + Collections.singletonMap(CLIENT_IP_KEY, ADDRESS), + passed.get(0)); + } + + @Test + public void Process_TakesNoKeysFromTheNativeEngine() throws Exception { + IPIntelligenceOnPremiseEngine engine = getWrapper().getEngine(); + EvidenceKeyFilter filter = engine.getEvidenceKeyFilter(); + for (int i = 0; i < PROCESSES; i++) { + try (FlowData data = + getWrapper().getPipeline().createFlowData()) { + data.addEvidence(CLIENT_IP_KEY, ADDRESS); + data.process(); + } + } + verify(nativeEngine, times(PROCESSES)) + .process(any(EvidenceIpiSwig.class)); + verify(nativeEngine, never()).getKeys(); + assertSame(filter, engine.getEvidenceKeyFilter()); + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ProcessReadsNoValueTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ProcessReadsNoValueTests.java new file mode 100644 index 000000000..6fb1ffa95 --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ProcessReadsNoValueTests.java @@ -0,0 +1,97 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.TestsBase; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EngineIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EvidenceIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.ResultsIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.StringValueSwig; +import fiftyone.ipintelligence.shared.IPIntelligenceData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +/** + * Processing keeps the native results and reads no value from them, and a + * value is read from them only when it is asked for. A stand-in for the + * native engine gives results that record what is asked of them. + */ +public class ProcessReadsNoValueTests extends TestsBase { + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.LowMemory); + } + + @After + public void cleanup() { + testCleanup(); + } + + @Test + public void Process_ReadsNoValue() throws Exception { + IPIntelligenceOnPremiseEngine engine = getWrapper().getEngine(); + Integer index = engine.getPropertyIndex("RegisteredName"); + assertNotNull(index); + EngineIpiSwig nativeEngine = mock(EngineIpiSwig.class); + ResultsIpiSwig results = mock(ResultsIpiSwig.class); + when(nativeEngine.process(any(EvidenceIpiSwig.class))) + .thenReturn(results); + when(results.containsProperty(anyString())).thenReturn(true); + StringValueSwig value = mock(StringValueSwig.class); + when(value.hasValue()).thenReturn(true); + when(value.getValue()).thenReturn("Example"); + when(results.getValueAsString(index.intValue())).thenReturn(value); + + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField("engine"); + field.setAccessible(true); + Object real = field.get(engine); + field.set(engine, nativeEngine); + try (FlowData data = getWrapper().getPipeline().createFlowData()) { + data.addEvidence("query.client-ip", "116.154.188.222"); + data.process(); + + verify(nativeEngine).process(any(EvidenceIpiSwig.class)); + verifyNoInteractions(results); + + IPIntelligenceData ipi = data.get(IPIntelligenceData.class); + assertEquals("Example", ipi.getRegisteredName().getValue()); + verify(results).getValueAsString(index.intValue()); + verify(results, never()).getValueAsString(anyString()); + } finally { + field.set(engine, real); + } + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexLaterDataSetTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexLaterDataSetTests.java new file mode 100644 index 000000000..5651ca46a --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexLaterDataSetTests.java @@ -0,0 +1,114 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EngineIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.VectorStringSwig; +import org.junit.Test; + +import java.io.File; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assume.assumeTrue; + +/** + * The table of property indexes is made once, from the names the native + * engine gives when the engine is created, and is not made again from a + * later data set, even one whose native engine gives other names. The + * engine is built from the Lite file with no names, then its native engine + * is replaced by one the engine creates from the ASN data file. + */ +public class PropertyIndexLaterDataSetTests { + + private static final String ASN_IPI_DATA_FILE_NAME = + "51Degrees-IPIV4AsnIpiV41.ipi"; + + @Test + public void LaterDataSet_TableNotMadeAgain() throws Exception { + File asn; + try { + asn = getFilePath(ASN_IPI_DATA_FILE_NAME); + } + catch (IllegalArgumentException e) { + assumeTrue(e.getMessage(), false); + return; + } + try (IPIntelligenceOnPremiseEngine engine = + EngineNamesTests.build(null)) { + Map before = table(engine); + assertFalse("Expected a table from the first data set.", + before.isEmpty()); + + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField("engine"); + field.setAccessible(true); + EngineIpiSwig first = (EngineIpiSwig) field.get(engine); + field.set(engine, null); + try { + engine.refreshData( + engine.getDataFileMetaData().getIdentifier(), + Files.readAllBytes(asn.toPath())); + } + finally { + first.delete(); + } + + List laterNames; + EngineIpiSwig later = (EngineIpiSwig) field.get(engine); + try (VectorStringSwig names = later.getRequiredProperties()) { + laterNames = new ArrayList<>(names); + } + assertFalse( + "Expected the later data set to give names the first did " + + "not, so a table made again would differ.", + before.keySet().containsAll(laterNames)); + + assertEquals(before, table(engine)); + for (String name : laterNames) { + if (before.containsKey(name) == false) { + assertNull( + "'" + name + "' is only in the later data set.", + engine.getPropertyIndex(name)); + } + } + } + } + + @SuppressWarnings("unchecked") + private static Map table( + IPIntelligenceOnPremiseEngine engine) throws Exception { + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField("propertyIndexes"); + field.setAccessible(true); + return new TreeMap<>((Map) field.get(engine)); + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ReadByPositionTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ReadByPositionTests.java new file mode 100644 index 000000000..de73a7ba6 --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ReadByPositionTests.java @@ -0,0 +1,86 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.TestsBase; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.ResultsIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.StringValueSwig; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.*; + +/** + * A read of a property the engine has a position for goes by that + * position, and asks the native results nothing by name, not even whether + * they hold the property. + */ +public class ReadByPositionTests extends TestsBase { + + private static final Logger logger = mock(Logger.class); + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.MaxPerformance); + } + + @After + public void cleanup() { + testCleanup(); + } + + @Test + public void StringRead_ByPosition_NoLookupByName() throws Exception { + IPIntelligenceOnPremiseEngine engine = getWrapper().getEngine(); + Integer index = engine.getPropertyIndex("RegisteredName"); + assertNotNull(index); + try (FlowData flowData = + getWrapper().getPipeline().createFlowData()) { + ResultsIpiSwig results = mock(ResultsIpiSwig.class); + when(results.containsProperty(any(String.class))).thenReturn(true); + StringValueSwig value = mock(StringValueSwig.class); + when(value.hasValue()).thenReturn(true); + when(value.getValue()).thenReturn("Example"); + when(results.getValueAsString(index.intValue())).thenReturn(value); + IPIntelligenceDataHashDefault data = + new IPIntelligenceDataHashDefault( + logger, + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + data.setResults(results); + + assertEquals("Example", data.getRegisteredName().getValue()); + + verify(results).getValueAsString(index.intValue()); + verify(results, never()).containsProperty(any(String.class)); + } + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ResultsReadByOneThreadTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ResultsReadByOneThreadTests.java new file mode 100644 index 000000000..3ba77454b --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/ResultsReadByOneThreadTests.java @@ -0,0 +1,189 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.TestsBase; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EngineIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EvidenceIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.ResultsIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.StringValueSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.WeightedUTF8StringListValueSwig; +import fiftyone.ipintelligence.shared.IPIntelligenceData; +import fiftyone.ipintelligence.shared.IPIntelligenceDataBaseOnPremise; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.IWeightedValue; +import fiftyone.pipeline.engines.Constants; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.RETURNS_DEFAULTS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Several threads read one finished Element Data at once, through the + * typed getters and through getValues, and only one of them at a time reads + * from the native results. A stand-in for the native engine gives results + * that count the threads inside them, and each read of them takes a moment + * so that two reads at once would overlap. + */ +public class ResultsReadByOneThreadTests extends TestsBase { + + private static final int THREADS = 4; + + private static final int READS = 10; + + private static final long READ_HOLD_MILLIS = 2; + + private static final long WAIT_SECONDS = 60; + + private static final Class[] WEIGHTED_STRINGS = + { List.class, IWeightedValue.class, String.class }; + + private final AtomicInteger inside = new AtomicInteger(); + + private final AtomicInteger most = new AtomicInteger(); + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.LowMemory); + } + + @After + public void cleanup() { + testCleanup(); + } + + @Test + public void SeveralThreads_OneReadsTheResultsAtATime() throws Exception { + IPIntelligenceOnPremiseEngine engine = getWrapper().getEngine(); + EngineIpiSwig nativeEngine = mock(EngineIpiSwig.class); + ResultsIpiSwig results = countingResults(); + when(nativeEngine.process(any(EvidenceIpiSwig.class))) + .thenReturn(results); + + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField("engine"); + field.setAccessible(true); + Object real = field.get(engine); + field.set(engine, nativeEngine); + ExecutorService executor = Executors.newFixedThreadPool(THREADS); + try (FlowData data = getWrapper().getPipeline().createFlowData()) { + data.addEvidence("query.client-ip", "116.154.188.222"); + data.process(); + IPIntelligenceData ipi = data.get(IPIntelligenceData.class); + + CountDownLatch start = new CountDownLatch(1); + List> readers = new ArrayList<>(); + for (int t = 0; t < THREADS; t++) { + boolean getter = t % 2 == 0; + readers.add(executor.submit(read(ipi, getter, start))); + } + start.countDown(); + for (Future reader : readers) { + assertEquals(Integer.valueOf(READS), + reader.get(WAIT_SECONDS, TimeUnit.SECONDS)); + } + assertTrue("Expected the stand-in results to be read.", + most.get() > 0); + assertEquals( + "More than one thread read the native results at once.", + 1, + most.get()); + } finally { + executor.shutdownNow(); + field.set(engine, real); + } + } + + private Callable read( + IPIntelligenceData ipi, + boolean getter, + CountDownLatch start) { + return () -> { + start.await(); + int done = 0; + for (int i = 0; i < READS; i++) { + if (getter) { + assertEquals("Example", + ipi.getRegisteredName().getValue()); + } + else { + ((IPIntelligenceDataBaseOnPremise) ipi).getValues( + "RegisteredName", WEIGHTED_STRINGS); + } + done++; + } + return done; + }; + } + + /** + * Results that count the threads inside any of their methods, holding + * each call for a moment. + */ + private ResultsIpiSwig countingResults() { + StringValueSwig value = mock(StringValueSwig.class); + when(value.hasValue()).thenReturn(true); + when(value.getValue()).thenReturn("Example"); + WeightedUTF8StringListValueSwig list = + mock(WeightedUTF8StringListValueSwig.class); + when(list.hasValue()).thenReturn(false); + when(list.getNoValueMessage()).thenReturn("No value"); + return mock(ResultsIpiSwig.class, invocation -> { + int now = inside.incrementAndGet(); + most.accumulateAndGet(now, Math::max); + try { + Thread.sleep(READ_HOLD_MILLIS); + switch (invocation.getMethod().getName()) { + case "containsProperty": + return true; + case "getValueAsString": + return value; + case "getValuesAsWeightedUTF8StringList": + return list; + default: + return RETURNS_DEFAULTS.answer(invocation); + } + } + finally { + inside.decrementAndGet(); + } + }); + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/WritesAfterProcessingTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/WritesAfterProcessingTests.java new file mode 100644 index 000000000..c1868c4e4 --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/WritesAfterProcessingTests.java @@ -0,0 +1,99 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.TestsBase; +import fiftyone.ipintelligence.shared.IPIntelligenceData; +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.data.AspectDataBase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.function.ThrowingRunnable; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; + +/** + * The check, for the Flow Element in this module, the + * {@link IPIntelligenceOnPremiseEngine}, that processes it, then writes to + * its Element Data and expects the error. Values the Element Data keeps once + * it has read them from the results are not writes, so reading every value + * still works. + * @see Specification + */ +public class WritesAfterProcessingTests extends TestsBase { + + @Before + public void init() throws Exception { + testInitialize(Constants.PerformanceProfiles.LowMemory); + } + + @After + public void cleanup() { + testCleanup(); + } + + private static void assertFails(String what, ThrowingRunnable write) { + assertThrows(what, IllegalStateException.class, write); + } + + /** + * Every write method of Element Data, and of Aspect Data, fails once its + * Flow Element has finished. + * @param data the Element Data of a finished Flow Element + */ + static void assertWritesFail(ElementData data) { + assertFails("put", () -> data.put("after", "value")); + assertFails("populateFromMap", () -> data.populateFromMap( + Collections.singletonMap("after", "value"))); + if (data instanceof AspectDataBase) { + AspectDataBase aspect = (AspectDataBase) data; + assertFails("setCloudRequestFailed", aspect::setCloudRequestFailed); + assertFails("addEngine", () -> aspect.addEngine(null)); + assertFails("addProcessCallable", + () -> aspect.addProcessCallable(null)); + } + } + + @Test + public void ipIntelligenceOnPremiseEngine_ElementDataRefusesWrites() + throws Exception { + try (FlowData data = getWrapper().getPipeline().createFlowData()) { + data.addEvidence("query.client-ip", "116.154.188.222"); + data.process(); + assertEquals(String.valueOf(data.getErrors()), + 0, data.getErrors().size()); + + IPIntelligenceData own = + data.getFromElement(getWrapper().getEngine()); + assertFalse(own.asKeyMap().isEmpty()); + assertWritesFail(own); + } + } +} diff --git a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/IPIntelligenceDataBaseOnPremise.java b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/IPIntelligenceDataBaseOnPremise.java index 32b2729e2..8a043be37 100644 --- a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/IPIntelligenceDataBaseOnPremise.java +++ b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/IPIntelligenceDataBaseOnPremise.java @@ -32,6 +32,7 @@ import fiftyone.pipeline.engines.data.AspectPropertyMetaData; import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; +import fiftyone.pipeline.engines.data.ProvidesValuesAsString; import fiftyone.pipeline.engines.flowelements.AspectEngine; import fiftyone.pipeline.engines.services.MissingPropertyService; import org.slf4j.Logger; @@ -40,6 +41,7 @@ import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.TreeMap; @@ -47,8 +49,15 @@ /** * Base class used for all 51Degrees on-premise IP Intelligence results classes. + *

    + * An instance reads its values from native results, and reading a value + * writes into them, so an instance is not safe to use from several threads + * at once. The caller is responsible for thread-safe access, for example by + * wrapping its calls in a lock of its own. See the + * thread safety specification. */ -public abstract class IPIntelligenceDataBaseOnPremise extends IPIntelligenceDataBase { +public abstract class IPIntelligenceDataBaseOnPremise extends IPIntelligenceDataBase + implements ProvidesValuesAsString { protected final Map, Class> primitiveTypes; private final Object dataLock = new Object(); @@ -99,6 +108,24 @@ public abstract AspectPropertyValue> getValues( protected abstract AspectPropertyValue getValueAsString( String propertyName); + /** + * {@inheritDoc} + *

    + * The Property is first read as {@link #get(String)} reads it, so a + * Property that is not in the results throws, and a lazy loaded result + * is waited for, exactly as for that accessor. + *

    + * The value is read from the results instance with no lock, so this + * instance is not safe to use from several threads at once. A caller + * that shares it coordinates its own access. + */ + @Override + public AspectPropertyValue getStoredValueAsString( + String propertyName) { + get(propertyName); + return getValueAsString(propertyName); + } + /** * Get the value for the specified property as an {@link Integer}. If the * property cannot be represented as an {@link Integer}, then the result @@ -229,12 +256,12 @@ public Map asKeyMap() { final Map keyedProps = allProps.get(key); final java.util.Collection relevantProps = keyedProps.values(); for (ElementPropertyMetaData property : relevantProps) { - map.put(property.getName().toLowerCase(), + map.put(property.getName().toLowerCase(Locale.ROOT), getAs(property.getName(), AspectPropertyValue.class, toWeightedListType(property.getType(), false))); } - populateFromMap(map); + keepValues(map); mapPopulated = true; } } diff --git a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java index ded023de3..930553734 100644 --- a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java +++ b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java @@ -32,7 +32,6 @@ import java.nio.file.Path; import java.util.Objects; import java.util.Optional; -import java.util.concurrent.atomic.AtomicBoolean; import static fiftyone.pipeline.util.FileFinder.getFilePath; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; @@ -47,18 +46,13 @@ public class FileUtils { public static final String IPI_PATH_VARIABLE = "_51DEGREES_IPI_PATH"; /** - * The old name of {@link #IPI_PATH_VARIABLE}, read only when that one - * is not set. + * The old name of {@link #IPI_PATH_VARIABLE}. It is no longer read. * @deprecated use {@link #IPI_PATH_VARIABLE}, and set the variable it * names in place of this one. */ @Deprecated public static final String IPI_PATH_ENV_VAR = "51DEGREES_IPI_PATH"; - // Set once the use of the deprecated name has been reported. - private static final AtomicBoolean OLD_NAME_REPORTED = - new AtomicBoolean(false); - public static final String ENTERPRISE_IPI_DATA_FILE_NAME = "Enterprise-IpIntelligenceV41.ipi"; public static final String ENTERPRISE_IPI_V41_DATA_FILE_NAME = "51Degrees-EnterpriseIpiV41.ipi"; @@ -114,26 +108,12 @@ public static File getHashFile() { /** * Check the aligned {@link #IPI_PATH_VARIABLE} environment variable and - * system property for an explicit path to the IPI data file, and when - * neither is set the deprecated {@link #IPI_PATH_ENV_VAR}, whose first - * use is reported on standard error. + * system property for an explicit path to the IPI data file. * * @return a file if the path is set and the file exists, otherwise null */ private static File getExplicitHashFile() { String explicitPath = environmentOrProperty(IPI_PATH_VARIABLE); - if (Objects.isNull(explicitPath)) { - explicitPath = environmentOrProperty(IPI_PATH_ENV_VAR); - if (Objects.nonNull(explicitPath) - && OLD_NAME_REPORTED.compareAndSet(false, true)) { - System.err.println( - "The environment variable " + IPI_PATH_ENV_VAR - + " is deprecated and " + IPI_PATH_VARIABLE - + " replaces it. Set " + IPI_PATH_VARIABLE - + " to the same value, as a later release may no longer " - + "read " + IPI_PATH_ENV_VAR + "."); - } - } if (Objects.nonNull(explicitPath)) { File explicitFile = new File(explicitPath); if (explicitFile.exists()) { diff --git a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtils.java b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtils.java index 75533a5ef..e9d94c26c 100644 --- a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtils.java +++ b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtils.java @@ -23,63 +23,47 @@ package fiftyone.ipintelligence.shared.testhelpers; import java.util.Base64; +import java.util.Locale; import java.util.Objects; -import java.util.concurrent.atomic.AtomicBoolean; /** * Helpers to obtain keys from the environment */ public class KeyUtils { /** - * Name of the aligned environment variable or system property which - * supplies the resource key. This name is checked before any legacy name. + * Name of the environment variable or system property which supplies + * the resource key. It is the only name read. */ public static final String RESOURCE_KEY_VARIABLE = "_51DEGREES_RESOURCE_KEY"; /** - * The old name of {@link #RESOURCE_KEY_VARIABLE}, read only when that - * one is not set. + * The old name of {@link #RESOURCE_KEY_VARIABLE}. It is no longer read. * @deprecated use {@link #RESOURCE_KEY_VARIABLE}, and set the variable * it names in place of this one. */ @Deprecated public static final String RESOURCE_KEY_ENV_VAR = "51DEGREES_RESOURCE_KEY"; - // Set once the use of the deprecated name has been reported. - private static final AtomicBoolean OLD_NAME_REPORTED = - new AtomicBoolean(false); - /** - * Obtain a resource key from an environment variable or a system property. - *

    - * The aligned name {@link #RESOURCE_KEY_VARIABLE} is checked first. When - * it is not set or is empty the deprecated {@link #RESOURCE_KEY_ENV_VAR} - * is checked, and its first use is reported on standard error. The - * supplied legacy name is checked last. Each name is looked up using + * Obtain the resource key from the environment variable or system + * property {@link #RESOURCE_KEY_VARIABLE}, looked up using * {@link #getNamedKey(String)}. - * @param legacyKeyName legacy name to try when the aligned name is not set - * @return the resource key, or null if no name is set + * @return the resource key, or null if it is not set + */ + public static String getResourceKey() { + return getNamedKey(RESOURCE_KEY_VARIABLE); + } + + /** + * Obtain the resource key from {@link #RESOURCE_KEY_VARIABLE}. The name + * passed is no longer read. + * @param legacyKeyName an old name, which is not read + * @return the resource key, or null if it is not set + * @deprecated use {@link #getResourceKey()} */ + @Deprecated public static String getResourceKey(String legacyKeyName) { - String resourceKey = getNamedKey(RESOURCE_KEY_VARIABLE); - if (Objects.isNull(resourceKey) || resourceKey.isEmpty()) { - String oldNameKey = getNamedKey(RESOURCE_KEY_ENV_VAR); - if (Objects.nonNull(oldNameKey) && oldNameKey.isEmpty() == false) { - if (OLD_NAME_REPORTED.compareAndSet(false, true)) { - System.err.println( - "The environment variable " + RESOURCE_KEY_ENV_VAR - + " is deprecated and " + RESOURCE_KEY_VARIABLE - + " replaces it. Set " + RESOURCE_KEY_VARIABLE - + " to the same value, as a later release may no longer " - + "read " + RESOURCE_KEY_ENV_VAR + "."); - } - return oldNameKey; - } - } - if (Objects.isNull(resourceKey)) { - resourceKey = getNamedKey(legacyKeyName); - } - return resourceKey; + return getResourceKey(); } /** @@ -90,7 +74,7 @@ public static String getResourceKey(String legacyKeyName) { public static String getNamedKey(String keyName) { String resourceKey = System.getenv(keyName); if (Objects.isNull(resourceKey)) { - resourceKey = System.getenv(keyName.toUpperCase()); + resourceKey = System.getenv(keyName.toUpperCase(Locale.ROOT)); if (Objects.isNull(resourceKey)) { resourceKey = System.getProperty(keyName); } diff --git a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/FileUtilsTest.java b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/FileUtilsTest.java index b1478996b..b45624e9b 100644 --- a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/FileUtilsTest.java +++ b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/FileUtilsTest.java @@ -57,40 +57,18 @@ public void testUserDir() throws IOException { * variable. */ @Test - public void oldPathNameIsReadWhenNewIsNotSet() throws IOException { - assumePathNamesNotInEnvironment(); - String savedNew = System.clearProperty(NEW_PATH_NAME); - String savedOld = System.clearProperty(OLD_PATH_NAME); - File file = File.createTempFile("old-path-name", ".ipi"); - try { - System.setProperty(OLD_PATH_NAME, file.getPath()); - assertEquals(file, FileUtils.getHashFile()); - } finally { - restore(NEW_PATH_NAME, savedNew); - restore(OLD_PATH_NAME, savedOld); - file.delete(); - } - } - - @Test - public void newPathNameWinsOverOld() throws IOException { - assumePathNamesNotInEnvironment(); + public void newPathNameIsRead() throws IOException { + assumeTrue(System.getenv(NEW_PATH_NAME) == null); String savedNew = System.clearProperty(NEW_PATH_NAME); - String savedOld = System.clearProperty(OLD_PATH_NAME); File newFile = File.createTempFile("new-path-name", ".ipi"); - File oldFile = File.createTempFile("old-path-name", ".ipi"); try { System.setProperty(NEW_PATH_NAME, newFile.getPath()); - System.setProperty(OLD_PATH_NAME, oldFile.getPath()); assertEquals(newFile, FileUtils.getHashFile()); } finally { restore(NEW_PATH_NAME, savedNew); - restore(OLD_PATH_NAME, savedOld); newFile.delete(); - oldFile.delete(); } } - @Test @SuppressWarnings("deprecation") public void oldPathConstantKeepsItsValue() throws Exception { @@ -101,11 +79,6 @@ public void oldPathConstantKeepsItsValue() throws Exception { FileUtils.class.getField("IPI_PATH_VARIABLE").get(null)); } - private static void assumePathNamesNotInEnvironment() { - assumeTrue(System.getenv(NEW_PATH_NAME) == null); - assumeTrue(System.getenv(OLD_PATH_NAME) == null); - } - private static void restore(String name, String value) { if (value == null) { System.clearProperty(name); diff --git a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtilsTest.java b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtilsTest.java index 03a7d5341..a0859f388 100644 --- a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtilsTest.java +++ b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtilsTest.java @@ -43,33 +43,37 @@ public void testGetEnv(){ * variable. */ @Test - public void oldKeyNameIsReadWhenNewIsNotSet() { - assumeKeyNamesNotInEnvironment(); + public void newKeyNameIsRead() { + assumeTrue(System.getenv(NEW_KEY_NAME) == null); String savedNew = System.clearProperty(NEW_KEY_NAME); - String savedOld = System.clearProperty(OLD_KEY_NAME); try { - System.setProperty(OLD_KEY_NAME, "old-name-key"); - assertEquals("old-name-key", - KeyUtils.getResourceKey("NoSuchLegacyName")); + System.setProperty(NEW_KEY_NAME, "new-name-key"); + assertEquals("new-name-key", + KeyUtils.getResourceKey()); } finally { restore(NEW_KEY_NAME, savedNew); - restore(OLD_KEY_NAME, savedOld); } } + /** + * A name the caller passes is not read in place of + * {@link KeyUtils#RESOURCE_KEY_VARIABLE}. + */ @Test - public void newKeyNameWinsOverOld() { - assumeKeyNamesNotInEnvironment(); + @SuppressWarnings("deprecation") + public void passedNameIsNotRead() { + assumeTrue(System.getenv(NEW_KEY_NAME) == null); + String passed = "AnyOtherResourceKeyName"; String savedNew = System.clearProperty(NEW_KEY_NAME); - String savedOld = System.clearProperty(OLD_KEY_NAME); try { + System.setProperty(passed, "other-name-key"); + assertNull(KeyUtils.getResourceKey(passed)); + assertNull(KeyUtils.getResourceKey()); System.setProperty(NEW_KEY_NAME, "new-name-key"); - System.setProperty(OLD_KEY_NAME, "old-name-key"); - assertEquals("new-name-key", - KeyUtils.getResourceKey("NoSuchLegacyName")); + assertEquals("new-name-key", KeyUtils.getResourceKey()); } finally { + System.clearProperty(passed); restore(NEW_KEY_NAME, savedNew); - restore(OLD_KEY_NAME, savedOld); } } @@ -83,11 +87,6 @@ public void oldKeyConstantKeepsItsValue() throws Exception { KeyUtils.class.getField("RESOURCE_KEY_VARIABLE").get(null)); } - private static void assumeKeyNamesNotInEnvironment() { - assumeTrue(System.getenv(NEW_KEY_NAME) == null); - assumeTrue(System.getenv(OLD_KEY_NAME) == null); - } - private static void restore(String name, String value) { if (value == null) { System.clearProperty(name); diff --git a/ip-intelligence/ip-intelligence.translation/src/main/java/fiftyone/ipintelligence/translation/flowelements/CountryCodeTranslationEngineBuilder.java b/ip-intelligence/ip-intelligence.translation/src/main/java/fiftyone/ipintelligence/translation/flowelements/CountryCodeTranslationEngineBuilder.java index 0a23fe9aa..affeec009 100644 --- a/ip-intelligence/ip-intelligence.translation/src/main/java/fiftyone/ipintelligence/translation/flowelements/CountryCodeTranslationEngineBuilder.java +++ b/ip-intelligence/ip-intelligence.translation/src/main/java/fiftyone/ipintelligence/translation/flowelements/CountryCodeTranslationEngineBuilder.java @@ -24,6 +24,7 @@ import fiftyone.ipintelligence.translation.Constants; import fiftyone.ipintelligence.translation.data.CountryCodeTranslationData; +import fiftyone.pipeline.annotations.DefaultValue; import fiftyone.pipeline.annotations.ElementBuilder; import org.slf4j.ILoggerFactory; import org.slf4j.LoggerFactory; @@ -61,6 +62,7 @@ public CountryCodeTranslationEngineBuilder(ILoggerFactory loggerFactory) { * @param sourceElementDataKey the IP Intelligence element data key * @return this builder */ + @DefaultValue(Constants.IP_INTELLIGENCE_KEY) public CountryCodeTranslationEngineBuilder setSourceElementDataKey( String sourceElementDataKey) { this.sourceElementDataKey = sourceElementDataKey; diff --git a/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/BuilderDefaultValueTests.java b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/BuilderDefaultValueTests.java new file mode 100644 index 000000000..508924ca5 --- /dev/null +++ b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/BuilderDefaultValueTests.java @@ -0,0 +1,63 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.translation; + +import fiftyone.ipintelligence.translation.flowelements.CountryCodeTranslationEngineBuilder; +import fiftyone.pipeline.annotations.DefaultValue; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Every configuration setter of the country code translation engine + * builder gives its default value beside it, with {@link DefaultValue}. + * See + * reference-implementation-notes.md line 79: + * "In all cases, there SHOULD be some mechanism to specify the default + * values alongside configuration property setters." + */ +public class BuilderDefaultValueTests { + + @Test + public void builderSetters_GiveTheirDefaults() { + List missing = new ArrayList<>(); + for (Method method : + CountryCodeTranslationEngineBuilder.class.getDeclaredMethods()) { + if (Modifier.isPublic(method.getModifiers()) && + method.getName().startsWith("set") && + method.isSynthetic() == false && + method.isBridge() == false && + method.getAnnotation(DefaultValue.class) == null) { + missing.add(method.getName() + + Arrays.toString(method.getParameterTypes())); + } + } + assertEquals(new ArrayList(), missing); + } +} diff --git a/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java index 3f5ab406b..fb7a7e24b 100644 --- a/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java +++ b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java @@ -50,8 +50,7 @@ * complete, localized, ordered country dropdown is produced. *

    * The data file is resolved from the {@code TestDataFile} system property, the - * {@code _51DEGREES_IPI_PATH} environment variable (or its deprecated - * name {@code 51DEGREES_IPI_PATH} when that is not set), or a few well known + * {@code _51DEGREES_IPI_PATH} environment variable, or a few well known * locations, and the test is skipped if none is found. *

    * The IP Intelligence element data key has differed between engine versions @@ -65,31 +64,12 @@ public class CountryTranslationIntegrationTest { private static final String SAMPLE_IP = "8.8.8.8"; private static final String IPI_PATH_VARIABLE = "_51DEGREES_IPI_PATH"; - private static final String OLD_IPI_PATH_VARIABLE = "51DEGREES_IPI_PATH"; - - private static boolean oldNameReported = false; /** - * The path the environment gives, from the deprecated name when the - * name that replaces it is not set, saying so on standard error the - * first time. + * The path the environment gives. */ private static String pathFromEnvironment() { - String path = System.getenv(IPI_PATH_VARIABLE); - if (path == null || path.isEmpty()) { - path = System.getenv(OLD_IPI_PATH_VARIABLE); - if (path != null && path.isEmpty() == false - && oldNameReported == false) { - oldNameReported = true; - System.err.println( - "The environment variable " + OLD_IPI_PATH_VARIABLE - + " is deprecated and " + IPI_PATH_VARIABLE - + " replaces it. Set " + IPI_PATH_VARIABLE - + " to the same value, as a later release may no longer " - + "read " + OLD_IPI_PATH_VARIABLE + "."); - } - } - return path; + return System.getenv(IPI_PATH_VARIABLE); } private static String resolveDataFile() { diff --git a/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/WritesAfterProcessingTests.java b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/WritesAfterProcessingTests.java new file mode 100644 index 000000000..ef21d4f5f --- /dev/null +++ b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/WritesAfterProcessingTests.java @@ -0,0 +1,91 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.translation; + +import fiftyone.ipintelligence.translation.data.ICountriesTranslationData; +import fiftyone.ipintelligence.translation.data.ICountryCodeTranslationData; +import fiftyone.ipintelligence.translation.flowelements.CountriesTranslationEngine; +import fiftyone.ipintelligence.translation.flowelements.CountryCodeTranslationEngine; +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.data.AspectDataBase; +import org.junit.Test; +import org.junit.function.ThrowingRunnable; + +import java.util.Collections; + +import static fiftyone.ipintelligence.translation.testhelpers.Fixtures.apv; +import static fiftyone.ipintelligence.translation.testhelpers.Fixtures.build; +import static fiftyone.ipintelligence.translation.testhelpers.Fixtures.weightify; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +/** + * The check, for the Flow Elements in this module, that processes each one, + * then writes to its Element Data and expects the error: the + * {@link CountryCodeTranslationEngine} and the + * {@link CountriesTranslationEngine}, after a stand-in IP Intelligence + * engine. + * @see Specification + */ +public class WritesAfterProcessingTests { + + private static void assertFails(String what, ThrowingRunnable write) { + assertThrows(what, IllegalStateException.class, write); + } + + /** + * Every write method of Element Data, and of Aspect Data, fails once its + * Flow Element has finished. + * @param data the Element Data of a finished Flow Element + */ + static void assertWritesFail(ElementData data) { + assertFails("put", () -> data.put("after", "value")); + assertFails("populateFromMap", () -> data.populateFromMap( + Collections.singletonMap("after", "value"))); + if (data instanceof AspectDataBase) { + AspectDataBase aspect = (AspectDataBase) data; + assertFails("setCloudRequestFailed", aspect::setCloudRequestFailed); + assertFails("addEngine", () -> aspect.addEngine(null)); + assertFails("addProcessCallable", + () -> aspect.addProcessCallable(null)); + } + } + + @Test + public void translationEngines_ElementDataRefusesWrites() + throws Exception { + try (Pipeline pipeline = + build(apv(weightify("GB", "FR")), apv(weightify("GB", "FR"))); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.accept-language", "fr_FR"); + data.process(); + assertEquals(String.valueOf(data.getErrors()), + 0, data.getErrors().size()); + + assertWritesFail(data.get(ICountryCodeTranslationData.class)); + assertWritesFail(data.get(ICountriesTranslationData.class)); + } + } +} diff --git a/ip-intelligence/ip-intelligence/pom.xml b/ip-intelligence/ip-intelligence/pom.xml index 2baf5e0fd..2f59e3acd 100644 --- a/ip-intelligence/ip-intelligence/pom.xml +++ b/ip-intelligence/ip-intelligence/pom.xml @@ -12,7 +12,7 @@ ip-intelligence 51Degrees :: IP Intelligence - Provides IP Intelligence for the 51Degrees Pipeline API. This can be used as an alternative to legacy IP Intelligence services such as MaxMind or IPinfo. + IP Intelligence and IP geolocation for Java. Finds an IP address's likely location (country, region, town, and coordinates), connection type, and network. Works on-premise with a data file or in the cloud. Built on the 51Degrees Pipeline API. An alternative to MaxMind and IPinfo. https://github.com/51Degrees/51degrees-java diff --git a/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceCloudPipelineBuilder.java b/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceCloudPipelineBuilder.java index b19e90a08..1706cc0ee 100644 --- a/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceCloudPipelineBuilder.java +++ b/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceCloudPipelineBuilder.java @@ -93,6 +93,10 @@ public Pipeline build() throws Exception { if (licenseKey != null && licenseKey.isEmpty() == false) { cloudRequestEngineBuilder.setLicenseKey(licenseKey); } + if (requestedProperties.isEmpty() == false) { + cloudRequestEngineBuilder.setRequestedProperties( + requestedProperties); + } if (cloudRequestOrigin != null && cloudRequestOrigin.isEmpty() == false) { cloudRequestEngineBuilder.setCloudRequestOrigin(cloudRequestOrigin); } diff --git a/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java b/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java index 210fa80dd..52f455e39 100644 --- a/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java +++ b/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java @@ -24,7 +24,9 @@ import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngineBuilder; +import fiftyone.pipeline.annotations.DefaultValue; import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.Constants; import fiftyone.pipeline.engines.data.AspectData; @@ -47,7 +49,9 @@ public class IPIntelligenceOnPremisePipelineBuilder extends PrePackagedPipelineBuilderBase { - protected boolean shareUsageEnabled = true; + protected boolean shareUsageEnabled = + fiftyone.pipeline.engines.fiftyone.flowelements.Constants + .SHARE_USAGE_DEFAULT_ENABLED; private String filename; private boolean createTempDataCopy; private int concurrency = -1; @@ -112,6 +116,8 @@ IPIntelligenceOnPremisePipelineBuilder setFilename( * @param enabled True to enable usage sharing. False to disable. * @return This builder instance. */ + @DefaultValue(booleanValue = fiftyone.pipeline.engines.fiftyone + .flowelements.Constants.SHARE_USAGE_DEFAULT_ENABLED) public IPIntelligenceOnPremisePipelineBuilder setShareUsage( boolean enabled) { shareUsageEnabled = enabled; @@ -126,6 +132,7 @@ public IPIntelligenceOnPremisePipelineBuilder setShareUsage( * @param enabled True to enable auto update. False to disable. * @return This builder instance. */ + @DefaultValue(booleanValue = Constants.DEFAULT_AUTOUPDATE_ENABLED) public IPIntelligenceOnPremisePipelineBuilder setAutoUpdate( boolean enabled) { autoUpdateEnabled = enabled; @@ -140,6 +147,7 @@ public IPIntelligenceOnPremisePipelineBuilder setAutoUpdate( * @param enabled True to enable file system watcher. False to disable. * @return This builder instance. */ + @DefaultValue(booleanValue = Constants.DEFAULT_WATCHER_ENABLED) public IPIntelligenceOnPremisePipelineBuilder setDataFileSystemWatcher( boolean enabled) { dataFileSystemWatcher = enabled; @@ -151,6 +159,7 @@ public IPIntelligenceOnPremisePipelineBuilder setDataFileSystemWatcher( * @param dataUpdateService an instance of a dataUpdateService * @return This builder instance. */ + @DefaultValue("The DataUpdateService given to the constructor") public IPIntelligenceOnPremisePipelineBuilder setDataUpdateService( DataUpdateService dataUpdateService) { this.dataUpdateService = dataUpdateService; @@ -159,7 +168,7 @@ public IPIntelligenceOnPremisePipelineBuilder setDataUpdateService( /** * Enable/Disable update on startup. - * Defaults to enabled. + * Defaults to disabled. * If enabled, the auto update system will be used to check for * an update before the IP Intelligence engine is created. * If an update is available, it will be downloaded and applied @@ -168,6 +177,7 @@ public IPIntelligenceOnPremisePipelineBuilder setDataUpdateService( * @param enabled True to enable update on startup. False to disable. * @return This builder instance. */ + @DefaultValue(booleanValue = Constants.DEFAULT_UPDATE_ON_STARTUP) public IPIntelligenceOnPremisePipelineBuilder setDataUpdateOnStartup( boolean enabled) { dataUpdateOnStartup = enabled; @@ -181,6 +191,7 @@ public IPIntelligenceOnPremisePipelineBuilder setDataUpdateOnStartup( * @param pollingIntervalSeconds The number of seconds between checks. * @return This builder instance. */ + @DefaultValue(intValue = Constants.DATA_UPDATE_POLLING_DEFAULT) public IPIntelligenceOnPremisePipelineBuilder setUpdatePollingInterval( int pollingIntervalSeconds) { @@ -194,6 +205,7 @@ public IPIntelligenceOnPremisePipelineBuilder setUpdatePollingInterval( * @param pollingIntervalMillis The number of milliseconds between checks. * @return This builder instance. */ + @DefaultValue(intValue = Constants.DATA_UPDATE_POLLING_DEFAULT * 1000) public IPIntelligenceOnPremisePipelineBuilder setUpdatePollingIntervalMillis( long pollingIntervalMillis) { updatePollingInterval = pollingIntervalMillis; @@ -208,6 +220,7 @@ public IPIntelligenceOnPremisePipelineBuilder setUpdatePollingIntervalMillis( * polling interval in seconds. * @return This builder instance. */ + @DefaultValue(intValue = Constants.DATA_UPDATE_RANDOMISATION_DEFAULT) public IPIntelligenceOnPremisePipelineBuilder setUpdateRandomisationMax( int randomisationMaxSeconds) { updateRandomisationMax = (long)randomisationMaxSeconds * 1000; @@ -222,6 +235,7 @@ public IPIntelligenceOnPremisePipelineBuilder setUpdateRandomisationMax( * polling interval in milliseconds. * @return This builder instance. */ + @DefaultValue(intValue = Constants.DATA_UPDATE_RANDOMISATION_DEFAULT * 1000) public IPIntelligenceOnPremisePipelineBuilder setUpdateRandomisationMaxMillis( long randomisationMaxMillis) { updateRandomisationMax = randomisationMaxMillis; @@ -235,6 +249,7 @@ public IPIntelligenceOnPremisePipelineBuilder setUpdateRandomisationMaxMillis( * @param key The license key. * @return This builder instance. */ + @DefaultValue("No license key") public IPIntelligenceOnPremisePipelineBuilder setDataUpdateLicenseKey( String key) { dataUpdateLicenseKey = key; @@ -248,6 +263,7 @@ public IPIntelligenceOnPremisePipelineBuilder setDataUpdateLicenseKey( * @param profile The performance profile to use. * @return This builder instance. */ + @DefaultValue("Balanced") public IPIntelligenceOnPremisePipelineBuilder setPerformanceProfile( Constants.PerformanceProfiles profile) { performanceProfile = profile; @@ -261,6 +277,7 @@ public IPIntelligenceOnPremisePipelineBuilder setPerformanceProfile( * @param concurrency expected concurrent accesses * @return this builder */ + @DefaultValue("The result of Runtime#getRuntime().getAvailableProcessors()") public IPIntelligenceOnPremisePipelineBuilder setConcurrency(int concurrency) { this.concurrency = concurrency; return this; @@ -272,6 +289,7 @@ public IPIntelligenceOnPremisePipelineBuilder setConcurrency(int concurrency) { * @param property the property that we want the engine to populate * @return this builder */ + @DefaultValue("All properties") public IPIntelligenceOnPremisePipelineBuilder setProperty(String property) { this.properties.add(property); return this; @@ -286,6 +304,7 @@ public IPIntelligenceOnPremisePipelineBuilder setProperty(String property) { * @param url the URL to check for a new data file * @return this builder */ + @DefaultValue("51Degrees default, set by the engine's builder") public IPIntelligenceOnPremisePipelineBuilder setDataUpdateUrl(String url) { dataUpdateUrl = url; return this; @@ -302,6 +321,7 @@ public IPIntelligenceOnPremisePipelineBuilder setDataUpdateUrl(String url) { * False otherwise * @return this builder */ + @DefaultValue(booleanValue = Constants.DEFAULT_VERIFY_MD5) public IPIntelligenceOnPremisePipelineBuilder setDataUpdateVerifyMd5(Boolean verify) { dataUpdateVerifyMd5 = verify; return this; @@ -315,6 +335,7 @@ public IPIntelligenceOnPremisePipelineBuilder setDataUpdateVerifyMd5(Boolean ver * @param formatter the formatter to use * @return this builder */ + @DefaultValue("51Degrees default, set by the engine's builder") public IPIntelligenceOnPremisePipelineBuilder setDataUpdateUrlFormatter(DataUpdateUrlFormatter formatter) { dataUpdateUrlFormatter = formatter; return this; @@ -333,22 +354,33 @@ public Pipeline build() throws Exception { new IPIntelligenceOnPremiseEngineBuilder(loggerFactory, dataUpdateService); ipIntelligenceEngine = configureAndBuild(engineBuilder); + List> made = new ArrayList<>(); if (ipIntelligenceEngine != null) { - // Add the share usage element to the list if enabled - if (shareUsageEnabled) { - getFlowElements().add( - new ShareUsageBuilder(loggerFactory).build()); - } // Add the IP Intelligence engine to the list + made.add(ipIntelligenceEngine); getFlowElements().add(ipIntelligenceEngine); + // Usage sharing goes last, so that it shares what every other + // Flow Element has added to the Evidence. + if (shareUsageEnabled) { + FlowElement shareUsage = + new ShareUsageBuilder(loggerFactory).build(); + made.add(shareUsage); + getFlowElements().add(shareUsage); + } } else { throw new RuntimeException("Unexpected error creating IP Intelligence engine."); } setAutoCloseElements(true); - // Create and return the pipeline - return super.build(); + // Create and return the pipeline, closing what this builder made if + // the Pipeline is refused. + try { + return super.build(); + } catch (Exception e) { + closeMadeElements(made, e); + throw e; + } } /** diff --git a/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligencePipelineBuilder.java b/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligencePipelineBuilder.java index 9d8cb7d2f..c71e48141 100644 --- a/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligencePipelineBuilder.java +++ b/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligencePipelineBuilder.java @@ -110,7 +110,7 @@ public IPIntelligenceOnPremisePipelineBuilder useOnPremise( /** * Use the 51Degrees Cloud service to perform IP Intelligence. * @param resourceKey The resource key to use when querying the Cloud service. - * Obtain one from https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=ip-intelligence-src-main-java-fiftyone-ipintelligence-ipintelligencepipelinebuilder.java&utm_term=usecloud + * Obtain one from https://51degrees.com/configure?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=ip-intelligence-src-main-java-fiftyone-ipintelligence-ipintelligencepipelinebuilder.java&utm_term=usecloud * @return A builder that can be used to configure and build a pipeline * that will use the Cloud IP Intelligence engine. */ diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/BuilderDefaultValueTests.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/BuilderDefaultValueTests.java new file mode 100644 index 000000000..96ee86770 --- /dev/null +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/BuilderDefaultValueTests.java @@ -0,0 +1,70 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence; + +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngineBuilder; +import fiftyone.pipeline.annotations.DefaultValue; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Every configuration setter of the IP Intelligence on-premise builders + * gives its default value beside it, with {@link DefaultValue}. See + * reference-implementation-notes.md line 79: + * "In all cases, there SHOULD be some mechanism to specify the default + * values alongside configuration property setters." + */ +public class BuilderDefaultValueTests { + + static List missingDefaults(Class... builders) { + List missing = new ArrayList<>(); + for (Class builder : builders) { + for (Method method : builder.getDeclaredMethods()) { + if (Modifier.isPublic(method.getModifiers()) && + method.getName().startsWith("set") && + method.isSynthetic() == false && + method.isBridge() == false && + method.getAnnotation(DefaultValue.class) == null) { + missing.add(builder.getSimpleName() + "#" + + method.getName() + + Arrays.toString(method.getParameterTypes())); + } + } + } + return missing; + } + + @Test + public void builderSetters_GiveTheirDefaults() { + assertEquals(new ArrayList(), missingDefaults( + IPIntelligenceOnPremisePipelineBuilder.class, + IPIntelligenceOnPremiseEngineBuilder.class)); + } +} diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/CloudOnPremiseSubstitutionTests.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/CloudOnPremiseSubstitutionTests.java new file mode 100644 index 000000000..352ebf27f --- /dev/null +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/CloudOnPremiseSubstitutionTests.java @@ -0,0 +1,86 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence; + +import fiftyone.ipintelligence.cloud.flowelements.IPIntelligenceCloudEngine; +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; +import fiftyone.ipintelligence.shared.IPIntelligenceData; +import fiftyone.pipeline.core.flowelements.FlowElement; +import org.junit.Test; + +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; + +/** + * The cloud and on-premise IP Intelligence engines put their results in + * Flow Data under the same key and as the same interface, so a consuming + * application reads them in the same way whichever engine is in the + * Pipeline. See + * README.md line 95: + * "Where both Cloud and On-Premise variants of an Engine are available, + * they MUST arrange that the Properties and values produced are compatible + * with each other so that the Engines can be substituted in the Pipeline + * without alteration to the consuming application." + */ +public class CloudOnPremiseSubstitutionTests { + + @Test + public void elementDataKey_IsTheSame() { + FlowElement cloud = + mock(IPIntelligenceCloudEngine.class, CALLS_REAL_METHODS); + FlowElement onPremise = + mock(IPIntelligenceOnPremiseEngine.class, CALLS_REAL_METHODS); + assertEquals( + onPremise.getElementDataKey(), + cloud.getElementDataKey()); + } + + @Test + public void elementData_IsIPIntelligenceData() { + assertTrue(IPIntelligenceData.class.isAssignableFrom( + elementDataType(IPIntelligenceCloudEngine.class))); + assertTrue(IPIntelligenceData.class.isAssignableFrom( + elementDataType(IPIntelligenceOnPremiseEngine.class))); + } + + private static Class elementDataType(Class engine) { + for (Class c = engine; c != null; c = c.getSuperclass()) { + Type type = c.getGenericSuperclass(); + if (type instanceof ParameterizedType) { + Type first = + ((ParameterizedType) type).getActualTypeArguments()[0]; + if (first instanceof Class) { + return (Class) first; + } + } + } + assertNotNull("No Element Data type found for " + engine, null); + return null; + } +} diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceCloudPipelineBuilderTests.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceCloudPipelineBuilderTests.java new file mode 100644 index 000000000..7c68c499a --- /dev/null +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceCloudPipelineBuilderTests.java @@ -0,0 +1,249 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.services.HttpClient; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * The cloud Pipeline builder passes the keys and the requested properties + * to the Cloud Request Engine, so a Pipeline can authenticate on a license + * key alone. A fake cloud service records each data request. + */ +public class IPIntelligenceCloudPipelineBuilderTests { + + private static final String BASE = "https://cloud.example/api/v4/"; + private static final String LICENSE_KEY = "abcdefghijkl1234"; + private static final String RESOURCE_KEY = "resourcekey"; + private static final String FIRST = "ip.registeredname"; + private static final String SECOND = "ip.registeredcountry"; + private static final String PROPERTIES_RESPONSE = + "{\"Products\":{\"ip\":{\"DataTier\":\"t\",\"Properties\":" + + "[{\"Name\":\"RegisteredName\",\"Type\":\"String\"," + + "\"Category\":\"c\"}]}}}"; + private static final String DATA_RESPONSE = "{\"ip\":{}}"; + + /** Records every data request and answers from fixed responses. */ + private static class FakeCloud implements HttpClient { + final List dataRequests = + Collections.synchronizedList(new ArrayList()); + + @Override + public HttpURLConnection connect(URL url) { + return new HttpURLConnection(url) { + @Override + public void connect() { + } + + @Override + public void disconnect() { + } + + @Override + public boolean usingProxy() { + return false; + } + + @Override + public int getResponseCode() { + return 200; + } + }; + } + + @Override + public String postData( + HttpURLConnection connection, + Map headers, + byte[] data) { + return answer( + connection.getURL(), + data == null ? "" : + new String(data, StandardCharsets.UTF_8)); + } + + @Override + public String getResponseString(HttpURLConnection connection) { + return answer(connection.getURL(), ""); + } + + @Override + public String getResponseString( + HttpURLConnection connection, + Map headers) { + return answer(connection.getURL(), ""); + } + + private String answer(URL url, String body) { + String path = url.getPath(); + if (path.endsWith("accessibleproperties")) { + return PROPERTIES_RESPONSE; + } + if (path.endsWith("evidencekeys")) { + return "[\"query.client-ip\"]"; + } + dataRequests.add(body); + return DATA_RESPONSE; + } + } + + private FakeCloud cloud; + + @Before + public void setUp() { + cloud = new FakeCloud(); + } + + private IPIntelligenceCloudPipelineBuilder builder(String resourceKey) { + return new IPIntelligencePipelineBuilder( + LoggerFactory.getILoggerFactory(), cloud) + .useCloud(resourceKey) + .setEndPoint(BASE); + } + + /** Process one request and give the fields of the form it sent. */ + private Map process(IPIntelligenceCloudPipelineBuilder builder) + throws Exception { + try (Pipeline pipeline = builder.build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.client-ip", "8.8.8.8").process(); + } + assertEquals(1, cloud.dataRequests.size()); + return form(cloud.dataRequests.get(0)); + } + + private static Map form(String body) + throws UnsupportedEncodingException { + Map fields = new HashMap<>(); + for (String pair : body.split("&")) { + if (pair.isEmpty()) { + continue; + } + String[] parts = pair.split("=", 2); + fields.put( + URLDecoder.decode(parts[0], "UTF-8"), + URLDecoder.decode(parts.length > 1 ? parts[1] : "", "UTF-8")); + } + return fields; + } + + @Test + public void LicenseKeyAlone_RequestedProperties_SentAsValues() + throws Exception { + Map sent = process(builder(null) + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Arrays.asList(FIRST, SECOND))); + + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST + "," + SECOND, sent.get("values")); + assertFalse(sent.containsKey("resource")); + } + + @Test + public void LicenseKeyAlone_RequestedPropertiesAsText_SentAsValues() + throws Exception { + Map sent = process(builder("") + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(FIRST + ", " + SECOND)); + + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST + "," + SECOND, sent.get("values")); + assertFalse(sent.containsKey("resource")); + } + + @Test + public void BothKeys_RequestedProperties_SendsAllThree() + throws Exception { + Map sent = process(builder(RESOURCE_KEY) + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Collections.singletonList(FIRST))); + + assertEquals(RESOURCE_KEY, sent.get("resource")); + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST, sent.get("values")); + } + + @Test + public void ResourceKey_NoRequestedProperties_SendsNoValues() + throws Exception { + Map sent = process(builder(RESOURCE_KEY)); + + assertEquals(RESOURCE_KEY, sent.get("resource")); + assertFalse(sent.containsKey("license")); + assertFalse(sent.containsKey("values")); + } + + @Test + public void LicenseKeyAlone_NoRequestedProperties_Refused() + throws Exception { + try { + builder(null).setLicenseKey(LICENSE_KEY).build().close(); + fail("A license key with no requested properties was built."); + } catch (PipelineConfigurationException e) { + assertTrue(e.getMessage(), + e.getMessage().contains("answers 400")); + assertTrue(e.getMessage(), + e.getMessage().contains("setRequestedProperties")); + assertFalse(e.getMessage().contains(LICENSE_KEY)); + } + assertTrue(cloud.dataRequests.isEmpty()); + } + + @Test + public void SetRequestedProperties_Null_Refused() { + try { + builder(RESOURCE_KEY).setRequestedProperties((List) null); + fail("A null list was accepted."); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("properties")); + } + try { + builder(RESOURCE_KEY).setRequestedProperties((String) null); + fail("A null string was accepted."); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("properties")); + } + } +} diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderDefaultsTest.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderDefaultsTest.java new file mode 100644 index 000000000..cd414b08a --- /dev/null +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderDefaultsTest.java @@ -0,0 +1,47 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence; + +import fiftyone.pipeline.engines.fiftyone.flowelements.Constants; +import fiftyone.pipeline.engines.services.DataUpdateService; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +/** + * Usage sharing is on unless the builder is told otherwise, as + * {@link Constants#SHARE_USAGE_DEFAULT_ENABLED} says. + */ +public class IPIntelligenceOnPremisePipelineBuilderDefaultsTest { + + @Test + public void ShareUsage_NotSet_FollowsDefault() { + IPIntelligenceOnPremisePipelineBuilder builder = + new IPIntelligenceOnPremisePipelineBuilder( + mock(DataUpdateService.class)); + assertEquals( + Constants.SHARE_USAGE_DEFAULT_ENABLED, + builder.shareUsageEnabled); + } +} diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java index 9e37ada63..737c81db7 100644 --- a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java @@ -140,7 +140,9 @@ public void setShareUsage_True_SharedFlowElementAdded() throws Exception { .build()) { List flowElements = pipeline.getFlowElements(); assertEquals(2, flowElements.size()); - assertTrue(flowElements.get(0) instanceof ShareUsageElement); + // Usage sharing is the last Flow Element. + assertTrue(flowElements.get(flowElements.size() - 1) + instanceof ShareUsageElement); } } diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremiseEngineNamesTests.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremiseEngineNamesTests.java new file mode 100644 index 000000000..7c56d11be --- /dev/null +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremiseEngineNamesTests.java @@ -0,0 +1,110 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence; + +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.RequiredPropertiesConfigSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.VectorStringSwig; +import fiftyone.pipeline.core.flowelements.Pipeline; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static fiftyone.ipintelligence.shared.testhelpers.FileUtils.LITE_IPI_DATA_FILE_NAME; +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; + +/** + * The names the on-premise pipeline builder has its engine create the native + * engine with, read from the configuration the engine passes to it. + */ +public class OnPremiseEngineNamesTests { + + @Test + public void NoNames_NativeEngineCreatedWithNoNames() throws Exception { + assertEquals( + "A pipeline configured with no names should have its " + + "engine create the native engine with no names.", + Collections.emptyList(), + namesCreatedWith(Collections.emptyList())); + } + + @Test + public void Names_NativeEngineCreatedWithOnlyThoseNames() + throws Exception { + List names = Collections.singletonList("RegisteredName"); + assertEquals( + "A pipeline configured with names should have its engine " + + "create the native engine with those names and no others.", + names, + namesCreatedWith(names)); + } + + /** + * Build an on-premise pipeline with the names given, and give the names + * its engine creates the native engine with. + */ + private static List namesCreatedWith(List names) + throws Exception { + File file; + try { + file = getFilePath(LITE_IPI_DATA_FILE_NAME); + } + catch (IllegalArgumentException e) { + assumeTrue(e.getMessage(), false); + return null; + } + IPIntelligenceOnPremisePipelineBuilder builder = + new IPIntelligencePipelineBuilder( + LoggerFactory.getILoggerFactory()) + .useOnPremise(file.getAbsolutePath(), false) + .setShareUsage(false) + .setAutoUpdate(false) + .setDataFileSystemWatcher(false); + for (String name : names) { + builder.setProperty(name); + } + try (Pipeline pipeline = builder.build()) { + return namesCreatedWith( + pipeline.getElement(IPIntelligenceOnPremiseEngine.class)); + } + } + + private static List namesCreatedWith( + IPIntelligenceOnPremiseEngine engine) throws Exception { + Field field = IPIntelligenceOnPremiseEngine.class + .getDeclaredField("propertiesConfigSwig"); + field.setAccessible(true); + RequiredPropertiesConfigSwig config = + (RequiredPropertiesConfigSwig) field.get(engine); + try (VectorStringSwig names = config.getProperties()) { + return new ArrayList<>(names); + } + } +} diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/RefusedBuildTests.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/RefusedBuildTests.java new file mode 100644 index 000000000..e905613d8 --- /dev/null +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/RefusedBuildTests.java @@ -0,0 +1,164 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence; + +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.engines.fiftyone.flowelements.ShareUsageBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.ShareUsageElement; +import fiftyone.pipeline.engines.services.DataUpdateServiceDefault; +import fiftyone.pipeline.engines.services.HttpClientDefault; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.AbstractList; +import java.util.ArrayList; +import java.util.List; + +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.junit.Assume.assumeTrue; + +/** + * When the on-premise builder's Pipeline is refused, the builder closes the + * Flow Elements it made itself, because no caller holds them to close, and + * leaves the caller's own Flow Elements open. + * @see Specification + */ +@SuppressWarnings("rawtypes") +public class RefusedBuildTests { + + /** + * The free data file, which every build has. + */ + private static final String ASN_IPI_DATA_FILE_NAME = + "51Degrees-IPIV4AsnIpiV41.ipi"; + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + /** + * Records every Flow Element the builder adds to its own list, which are + * the ones it made. + */ + private static class RecordingBuilder + extends IPIntelligenceOnPremisePipelineBuilder { + private final List made = new ArrayList<>(); + + RecordingBuilder(ILoggerFactory loggerFactory) { + super( + loggerFactory, + new DataUpdateServiceDefault( + loggerFactory.getLogger( + DataUpdateServiceDefault.class.getName()), + new HttpClientDefault())); + } + + List elements() { + return super.getFlowElements(); + } + + @Override + protected List getFlowElements() { + final List elements = super.getFlowElements(); + return new AbstractList() { + @Override + public FlowElement get(int index) { + return elements.get(index); + } + + @Override + public int size() { + return elements.size(); + } + + @Override + public void add(int index, FlowElement element) { + made.add(element); + elements.add(index, element); + } + + @Override + public FlowElement set(int index, FlowElement element) { + return elements.set(index, element); + } + + @Override + public FlowElement remove(int index) { + return elements.remove(index); + } + }; + } + } + + private static File dataFile() { + try { + return getFilePath(ASN_IPI_DATA_FILE_NAME); + } + catch (IllegalArgumentException e) { + assumeTrue(e.getMessage(), false); + return null; + } + } + + /** + * Usage sharing the caller adds goes before the engine and the usage + * sharing element the builder adds, so the Pipeline is refused. + */ + @Test + public void Refused_ElementsTheBuilderMadeAreClosed() throws Exception { + RecordingBuilder builder = new RecordingBuilder(loggerFactory); + builder.setFilename(dataFile().getAbsolutePath(), false) + .setAutoUpdate(false) + .setDataFileSystemWatcher(false); + ShareUsageElement callers = + new ShareUsageBuilder(loggerFactory).build(); + try { + builder.addFlowElement(callers); + try { + builder.build().close(); + fail("A Pipeline with usage sharing first was built"); + } catch (PipelineConfigurationException e) { + assertTrue(e.getMessage(), + e.getMessage().contains("must be the last Flow Element")); + } + + assertFalse("The builder made no Flow Elements", + builder.made.isEmpty()); + for (FlowElement element : builder.made) { + assertTrue(element + " that the builder made is open", + element.isClosed()); + assertFalse(element + " is still in the builder's list", + builder.elements().contains(element)); + } + assertFalse("The caller's own Flow Element was closed", + callers.isClosed()); + } finally { + callers.close(); + } + } +} diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/UsageSharingLastTests.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/UsageSharingLastTests.java new file mode 100644 index 000000000..bcab7f2cc --- /dev/null +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/UsageSharingLastTests.java @@ -0,0 +1,109 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence; + +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.ShareUsageElement; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.List; + +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +/** + * The on-premise builder adds usage sharing after the engine and after any + * element the caller adds, as the last Flow Element of the Pipeline. + * @see Specification + */ +@SuppressWarnings("rawtypes") +public class UsageSharingLastTests { + + /** + * The free data file, which every build has. + */ + private static final String ASN_IPI_DATA_FILE_NAME = + "51Degrees-IPIV4AsnIpiV41.ipi"; + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + private static File dataFile() { + try { + return getFilePath(ASN_IPI_DATA_FILE_NAME); + } + catch (IllegalArgumentException e) { + assumeTrue(e.getMessage(), false); + return null; + } + } + + private IPIntelligenceOnPremisePipelineBuilder builder() + throws Exception { + return new IPIntelligencePipelineBuilder(loggerFactory) + .useOnPremise(dataFile().getAbsolutePath(), false) + .setAutoUpdate(false) + .setDataFileSystemWatcher(false) + .setShareUsage(true); + } + + private static void assertLast(List elements) { + assertTrue(elements.toString(), + elements.get(elements.size() - 1) instanceof ShareUsageElement); + } + + @Test + public void ShareUsageOn_UsageSharingIsLast() throws Exception { + try (Pipeline pipeline = builder().build()) { + List elements = pipeline.getFlowElements(); + assertEquals(2, elements.size()); + assertTrue(elements.toString(), + elements.get(0) instanceof IPIntelligenceOnPremiseEngine); + assertLast(elements); + } + } + + /** + * An element the caller adds after turning usage sharing on still goes + * before usage sharing, which the builder adds when it builds. + */ + @Test + public void ElementAddedAfterShareUsage_UsageSharingIsLast() + throws Exception { + try (Pipeline pipeline = builder() + .addFlowElement(new SequenceElementBuilder(loggerFactory).build()) + .build()) { + List elements = pipeline.getFlowElements(); + assertEquals(3, elements.size()); + assertLast(elements); + } + } +} diff --git a/ip-intelligence/pom.xml b/ip-intelligence/pom.xml index 1355b5932..20cd8b5cc 100644 --- a/ip-intelligence/pom.xml +++ b/ip-intelligence/pom.xml @@ -45,7 +45,7 @@ 20211205 3.9.0 - 3.26.0 + 3.28.0 1.6.0 2.5.2 3.2.2 @@ -218,13 +218,11 @@ ${maven.multiModuleProjectDirectory} - - ${TestResourceKey} - - ${SuperResourceKey} - - ${LicenseKey} + + <_51DEGREES_RESOURCE_KEY>${_51DEGREES_RESOURCE_KEY} + <_51DEGREES_LICENSE_KEY>${_51DEGREES_LICENSE_KEY} @@ -392,9 +390,11 @@ - - https://github.com/51Degrees/ip-intelligence-java - scm:git:git://github.com/51Degrees/ip-intelligence-java.git - scm:git:info@51degrees.com:51Degrees/ip-intelligence-java.git + + https://github.com/51Degrees/51degrees-java + scm:git:https://github.com/51Degrees/51degrees-java.git + scm:git:https://github.com/51Degrees/51degrees-java.git diff --git a/location/README.md b/location/README.md index b3f8f3db7..9d6a62201 100644 --- a/location/README.md +++ b/location/README.md @@ -18,7 +18,7 @@ but additional caution should be applied. You will require a [resource key](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=data) to use the Cloud API. You can create resource keys using our -[configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=data), see our +[configuration page](https://51degrees.com/configure/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=data), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=data) on how to use this. ## Installation @@ -77,3 +77,14 @@ For complete documentation on the Pipeline API and associated engines, see the [ [Documentation]: https://51degrees.com/documentation/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=project-documentation [maven]: https://central.sonatype.com/artifact/com.51degrees/geo-location +## Flow Data and threads + +A Flow Data cannot be used by two threads at once. The web examples handle +each request on one thread, and read the Flow Data there. An application that +hands work from a request to another thread or task reads the values that +work needs from the Flow Data on the request's thread, into its own +variables, and passes those values, not the Flow Data. The Flow Data is +closed when the request ends, so work that outlives the request must not +keep it either. The +[pipeline README](../pipeline/README.md#flow-data-and-threads) +shows such a hand-off. diff --git a/location/docs/Doxyfile b/location/docs/Doxyfile index 953e37e3b..7234413dd 100644 --- a/location/docs/Doxyfile +++ b/location/docs/Doxyfile @@ -163,7 +163,7 @@ INLINE_INHERITED_MEMB = NO # shortest path that makes the file name unique will be used # The default value is: YES. -FULL_PATH_NAMES = YES +FULL_PATH_NAMES = NO # The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. # Stripping is only done if one of the specified strings matches the left-hand @@ -1469,6 +1469,7 @@ DISABLE_INDEX = YES # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = YES +HTML_INDEX_NOINDEX = YES # The HIDE_TREEVIEW_ROOT tag is used to remove the root node from the tree view. # When not hidden, the tree root will be PROJECT_NAME. If the tree root is set diff --git a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java index 6c8eae436..8bec6d6f0 100644 --- a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java +++ b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java @@ -43,6 +43,8 @@ import org.json.JSONObject; import org.slf4j.Logger; +import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.NoCloudResponse; + import java.util.*; import org.json.JSONArray; @@ -135,7 +137,19 @@ public GeoLocationCloudEngine( protected void processEngine(FlowData data, CloudGeoData aspectData) { if (cloudRequestEngine != null) { CloudRequestData requestData = data.getFromElement(cloudRequestEngine); - String json = requestData.getJsonResponse(); + String json = getJsonResponse( + data, + aspectData, + requestData, + cloudRequestEngine); + + // A failed cloud request leaves no response. The Cloud Request + // Engine reports why, so this engine adds no error of its own. + if (json == null || json.isEmpty()) { + logger.warn(String.format( + NoCloudResponse, getClass().getSimpleName())); + return; + } JSONObject jsonObj = new JSONObject(json); JSONObject geoObj = getElementJson(jsonObj); @@ -143,7 +157,7 @@ protected void processEngine(FlowData data, CloudGeoData aspectData) { Map methodData = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); for (AspectPropertyMetaData property : getProperties()) { - String key = property.getName().toLowerCase(); + String key = property.getName().toLowerCase(Locale.ROOT); // A property the response leaves out has no entry, so // reading it throws a PropertyMissingException. if (geoObj.has(key) == false) { @@ -256,8 +270,10 @@ private boolean loadAspectProperties(CloudRequestEngine engine) { new ArrayList(), true, null, - item.delayExecution, - item.evidenceProperties); + item.delayExecution != null ? + item.delayExecution : false, + item.evidenceProperties != null ? + item.evidenceProperties : new ArrayList()); loaded.add(property); } diff --git a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/CloudRequestFailureRecordedTests.java b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/CloudRequestFailureRecordedTests.java new file mode 100644 index 000000000..f6056860c --- /dev/null +++ b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/CloudRequestFailureRecordedTests.java @@ -0,0 +1,158 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation.flowelements; + +import fiftyone.geolocation.core.Enums; +import fiftyone.geolocation.data.CloudGeoData; +import fiftyone.pipeline.cloudrequestengine.CloudRequestException; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngineBuilder; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.HttpClient; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.stubbing.Answer; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * A failed Cloud Request Engine request that the cloud location engine + * meets while the Flow Data is processed is recorded once, as the Cloud + * Request Engine's error and not the location engine's, and a property is + * then missing with the CloudRequestFailed reason. + * @see Specification + */ +public class CloudRequestFailureRecordedTests { + + private static final String PROPERTIES = + "{'Products': {'location': {'DataTier': 'tier', 'Properties': " + + "[{'Name': 'Town', 'Type': 'String', " + + "'Category': 'Location'}]}}}"; + + private static final String EVIDENCE_KEYS = "['query.User-Agent']"; + + private static final String FAILED = + "{\"errors\":[\"The request failed.\"]}"; + + /** + * A client that answers the discovery requests and fails the request + * for the Flow Data with an error response. + */ + private static HttpClient failingClient() throws Exception { + HttpClient client = mock(HttpClient.class); + when(client.connect(any(URL.class))).thenAnswer(invocation -> { + URL url = invocation.getArgument(0); + HttpURLConnection connection = mock(HttpURLConnection.class); + when(connection.getURL()).thenReturn(url); + when(connection.getResponseCode()).thenReturn(200); + when(connection.getHeaderFields()).thenReturn(new HashMap<>()); + return connection; + }); + Answer byUrl = invocation -> { + String path = ((HttpURLConnection) invocation.getArgument(0)) + .getURL().getPath(); + if (path.endsWith("properties")) { + return PROPERTIES; + } + if (path.endsWith("evidencekeys")) { + return EVIDENCE_KEYS; + } + return FAILED; + }; + when(client.getResponseString(any(HttpURLConnection.class))) + .thenAnswer(byUrl); + when(client.getResponseString( + any(HttpURLConnection.class), + ArgumentMatchers.anyMap())).thenAnswer(byUrl); + when(client.postData( + any(HttpURLConnection.class), + ArgumentMatchers.anyMap(), + any(byte[].class))).thenAnswer(byUrl); + return client; + } + + @Test + public void FailedRequest_RecordedOnceAsTheRequestEnginesError() + throws Exception { + Logger logger = mock(Logger.class); + ILoggerFactory loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + CloudRequestEngine requestEngine = + new CloudRequestEngineBuilder(loggerFactory, failingClient()) + .setResourceKey("resource_key") + .build(); + GeoLocationCloudEngine locationEngine = + new GeoLocationCloudEngineBuilder(loggerFactory) + .build(Enums.GeoLocationProvider.FiftyOneDegrees); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(true) + .addFlowElement(requestEngine) + .addFlowElement(locationEngine) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + + List errors = new ArrayList<>(data.getErrors()); + assertEquals(errors.toString(), 1, errors.size()); + assertSame( + "The failure was recorded as another element's error", + requestEngine, + errors.get(0).getFlowElement()); + assertTrue( + errors.get(0).getThrowable().toString(), + errors.get(0).getThrowable() instanceof CloudRequestException); + + CloudGeoData location = data.getFromElement(locationEngine); + try { + location.getTown(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + assertEquals( + e.getMessage(), + "CloudRequestFailed", + e.getReason().name()); + } + } + } +} diff --git a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineNullValueTests.java b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineNullValueTests.java index 8bfe09c1b..74391dba6 100644 --- a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineNullValueTests.java +++ b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineNullValueTests.java @@ -36,6 +36,7 @@ import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; import org.junit.Test; import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Arrays; @@ -46,8 +47,12 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.NoCloudResponse; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** @@ -142,6 +147,27 @@ public void SetNoValueReasons_ChangesNothing() throws Exception { } } + /** + * A failed cloud request leaves no response. The engine processes + * nothing, throws nothing and logs one warning each time, as the Cloud + * Request Engine's error says why. + */ + @Test + public void NoResponse_ProcessesNothing() throws Exception { + Logger logger = mock(Logger.class); + ILoggerFactory warnings = mock(ILoggerFactory.class); + when(warnings.getLogger(anyString())).thenReturn(logger); + try (GeoLocationCloudEngine engine = buildEngine(warnings)) { + for (String response : new String[] { null, "" }) { + CloudGeoData data = process(engine, response); + + assertFalse(data.asKeyMap().containsKey("Town")); + } + } + verify(logger, times(2)).warn(String.format( + NoCloudResponse, GeoLocationCloudEngine.class.getSimpleName())); + } + private static void assertMissing(Runnable read) { try { read.run(); @@ -183,8 +209,16 @@ private static void assertValue(Object expected, Object value) * the Suburb and Town properties. */ private GeoLocationCloudEngine buildEngine() throws Exception { + return buildEngine(loggerFactory); + } + + /** + * The same engine, logging through the given factory. + */ + private GeoLocationCloudEngine buildEngine(ILoggerFactory engineLoggers) + throws Exception { GeoLocationCloudEngine engine = - new GeoLocationCloudEngineBuilder(loggerFactory) + new GeoLocationCloudEngineBuilder(engineLoggers) .build(Enums.GeoLocationProvider.FiftyOneDegrees); ProductMetaData product = new ProductMetaData(); diff --git a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudRequestFailedTests.java b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudRequestFailedTests.java new file mode 100644 index 000000000..1fae1622a --- /dev/null +++ b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudRequestFailedTests.java @@ -0,0 +1,163 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation.flowelements; + +import fiftyone.geolocation.core.Enums; +import fiftyone.geolocation.data.CloudGeoData; +import fiftyone.geolocation.data.CloudGeoDataDefault; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * When the Cloud Request Engine's request for the Flow Data failed, a + * property of the location cloud engine is missing with the + * CloudRequestFailed reason, rather than a reason about the resource key. + */ +public class GeoLocationCloudRequestFailedTests { + + private ILoggerFactory loggerFactory; + + private CloudRequestEngine requestEngine; + + @Before + public void init() { + Logger logger = mock(Logger.class); + loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + } + + @Test + public void RequestFailedWithNoResponse_CloudRequestFailed() + throws Exception { + assertCloudRequestFailed(null); + } + + @Test + public void RequestFailedWithAnErrorResponse_CloudRequestFailed() + throws Exception { + assertCloudRequestFailed("{ \"errors\": [ \"an error\" ] }"); + } + + private void assertCloudRequestFailed(String response) throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + CloudGeoData data = process(engine, response); + try { + data.getTown(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + assertEquals( + e.getMessage(), + "CloudRequestFailed", + e.getReason().name()); + assertEquals( + "Property 'town' not found in data for element '" + + engine.getElementDataKey() + "'. This is because " + + "an upstream cloud request failed, so no data was " + + "available to populate the property.", + e.getMessage()); + } + } + } + + /** + * An engine whose Cloud Request Engine reports the location product, + * holding a property other than the one read. + */ + // thenReturn has a varargs overload, so a generic Map is an unchecked array. + @SuppressWarnings("unchecked") + private GeoLocationCloudEngine buildEngine() throws Exception { + GeoLocationCloudEngine engine = + new GeoLocationCloudEngineBuilder(loggerFactory) + .build(Enums.GeoLocationProvider.FiftyOneDegrees); + + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("country", "String", "cat", null)); + Map publicProperties = new HashMap<>(); + publicProperties.put(engine.getElementDataKey(), product); + + requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(publicProperties); + + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + engine.addPipeline(pipeline); + return engine; + } + + private CloudGeoData process( + GeoLocationCloudEngine engine, + String response) { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn(response); + when(cloudData.getProcessStarted()).thenReturn(true); + + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + when(flowData.getErrors()).thenReturn( + Collections.singletonList(new FlowError.Default( + new Exception("The cloud request failed."), + requestEngine))); + + CloudGeoData data = new CloudGeoDataDefault( + loggerFactory.getLogger(CloudGeoData.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + try { + engine.processEngine(flowData, data); + } catch (RuntimeException e) { + // A failed request's response may not be readable. The + // property is read afterwards either way, as it is through a + // Pipeline that suppresses process exceptions. + } + return data; + } +} diff --git a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/WritesAfterProcessingTests.java b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/WritesAfterProcessingTests.java new file mode 100644 index 000000000..d94cbdacb --- /dev/null +++ b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/WritesAfterProcessingTests.java @@ -0,0 +1,167 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation.flowelements; + +import fiftyone.geolocation.core.Enums; +import fiftyone.geolocation.data.CloudGeoData; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudAspectEngineBase; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.TryGetResult; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.FlowElement.DataFactory; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.engines.data.AspectDataBase; +import org.junit.Test; +import org.junit.function.ThrowingRunnable; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The check, for the Flow Element in this module, the + * {@link GeoLocationCloudEngine}, that processes it, then writes to its + * Element Data and expects the error. It is processed through its public + * process method with a fixed cloud response, so the check needs no + * resource key. + * @see Specification + */ +public class WritesAfterProcessingTests { + + private static final ILoggerFactory LOGGER_FACTORY = + LoggerFactory.getILoggerFactory(); + + private static void assertFails(String what, ThrowingRunnable write) { + assertThrows(what, IllegalStateException.class, write); + } + + /** + * Every write method of Element Data, and of Aspect Data, fails once its + * Flow Element has finished. + * @param data the Element Data of a finished Flow Element + */ + static void assertWritesFail(ElementData data) { + assertFails("put", () -> data.put("after", "value")); + assertFails("populateFromMap", () -> data.populateFromMap( + Collections.singletonMap("after", "value"))); + if (data instanceof AspectDataBase) { + AspectDataBase aspect = (AspectDataBase) data; + assertFails("setCloudRequestFailed", aspect::setCloudRequestFailed); + assertFails("addEngine", () -> aspect.addEngine(null)); + assertFails("addProcessCallable", + () -> aspect.addProcessCallable(null)); + } + } + + /** + * Attach the engine to a Pipeline whose Cloud Request Engine reports the + * product given. + */ + private static > T attach( + T engine, + String productKey, + PropertyMetaData property) { + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Collections.singletonList(property); + Map products = new HashMap<>(); + products.put(productKey, product); + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(products); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + engine.addPipeline(pipeline); + return engine; + } + + /** + * Process the engine through its public process method, with a Flow + * Data stand-in that holds the Element Data the engine adds and gives + * the Cloud Request Engine's data with the response given. + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + private static T process( + FlowElement engine, + String response) throws Exception { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getProcessStarted()).thenReturn(true); + when(cloudData.getJsonResponse()).thenReturn(response); + final FlowData flowData = mock(FlowData.class); + final List added = new ArrayList<>(); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + when(flowData.getOrAdd(any(TypedKey.class), any(DataFactory.class))) + .thenAnswer(invocation -> { + if (added.isEmpty()) { + DataFactory factory = invocation.getArgument(1); + added.add(factory.create(flowData)); + } + return added.get(0); + }); + when(flowData.tryGetValue(any(TypedKey.class))) + .thenAnswer(invocation -> { + TryGetResult result = new TryGetResult<>(); + if (added.isEmpty() == false) { + result.setValue(added.get(0)); + } + return result; + }); + engine.process(flowData); + assertEquals(1, added.size()); + return (T) added.get(0); + } + + @Test + public void geoLocationCloudEngine_ElementDataRefusesWrites() + throws Exception { + try (GeoLocationCloudEngine engine = attach( + new GeoLocationCloudEngineBuilder(LOGGER_FACTORY) + .build(Enums.GeoLocationProvider.FiftyOneDegrees), + "location", + new PropertyMetaData("town", "String", "cat", null))) { + CloudGeoData own = process( + engine, + "{ \"location\": { \"town\": \"Example\" } }"); + assertEquals("Example", own.getTown().getValue()); + assertWritesFail(own); + assertFails("setTown", () -> own.setTown(null)); + } + } +} diff --git a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java index c3088a99f..3db4b6fe6 100644 --- a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java +++ b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/CombiningServices.java @@ -41,7 +41,6 @@ import fiftyone.pipeline.engines.services.HttpClientDefault; import org.slf4j.ILoggerFactory; import org.slf4j.LoggerFactory; -import java.util.concurrent.Future; import static fiftyone.geolocation.core.Constants.EVIDENCE_GEO_LAT_PARAM_KEY; import static fiftyone.geolocation.core.Constants.EVIDENCE_GEO_LON_PARAM_KEY; @@ -128,14 +127,19 @@ public class CombiningServices { "/11D167 Safari/9537.53"; public static void main(String[] args) throws Exception { - // Obtain a resource key for free at https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=geo-location.shell.examples-src-main-java-fiftyone-geolocation-examples-combiningservices.java&utm_term=main + // Obtain a resource key for free at https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=geo-location.shell.examples-src-main-java-fiftyone-geolocation-examples-combiningservices.java&utm_term=main // Make sure to include the 'Country' and 'IsMobile' properties as they // are used by this example. - String resourceKey = "!!Your resource license key!!"; + String resourceKey = ResourceKey.find(args); - if (resourceKey.startsWith("!!")) { - System.out.println("You need to create a resource key at " + - "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=geo-location.shell.examples-src-main-java-fiftyone-geolocation-examples-combiningservices.java&utm_term=resource-key-required and paste it into this example."); + if (resourceKey == null) { + System.out.println("This example calls the 51Degrees cloud " + + "service, which needs a resource key, and none is set."); + System.out.println("Set the " + ResourceKey.VARIABLE + + " environment variable to a resource key, or give the key " + + "as the first argument, and run the example again."); + System.out.println("Create a resource key at " + + "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=geo-location.shell.examples-src-main-java-fiftyone-geolocation-examples-combiningservices.java&utm_term=resource-key-required"); System.out.println("Make sure to include the 'Country' and " + "'IsMobile' properties as they are used by this example."); } @@ -171,24 +175,13 @@ public static void main(String[] args) throws Exception { AspectPropertyValue isMobile = flowData.get(DeviceData.class).getIsMobile(); - Future future = flowData.get(GeoData.class).getProcessFuture(); - System.out.print("Awaiting response"); - outputUntilCancelled(".", 1000, future); - System.out.println(); System.out.println("Country: " + country.toString()); System.out.println("IsMobile: " + isMobile.toString()); } } - } - - private static void outputUntilCancelled( - String text, - int intervalMs, - Future future) throws InterruptedException { - while (future.isDone() == false) { - System.out.print(text); - Thread.sleep(intervalMs); - } - future.cancel(true); + FindOutMore.write( + System.out, + "https://51degrees.com/documentation/_reverse_geocoding__examples__combining_services.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=geo-location.shell.examples-combiningservices.java&utm_term=find-out-more", + "https://51degrees.com/location?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=geo-location.shell.examples-combiningservices.java&utm_term=find-out-more"); } } diff --git a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/FindOutMore.java b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/FindOutMore.java new file mode 100644 index 000000000..c4d878857 --- /dev/null +++ b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/FindOutMore.java @@ -0,0 +1,76 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation.examples; + +import java.io.PrintStream; + +/** + * Writes the "Find out more" section that ends each Location example's + * output, set apart by a rule above and below. It gives the example's page + * and the Location page on 51degrees.com, each tagged with the example, then + * the repository. + */ +public final class FindOutMore { + + /** + * The heading the section starts with. + */ + public static final String HEADING = "Find out more"; + + /** + * The rule above and below the section. + */ + public static final String RULE = + "----------------------------------------------------------------"; + + /** + * The repository the examples come from. + */ + public static final String REPOSITORY = + "https://github.com/51Degrees/51degrees-java"; + + private FindOutMore() { + } + + /** + * Writes the section. + * @param output where to write, flushed afterwards + * @param examplePage the address of the example's page, with its + * campaign tags + * @param locationPage the address of the Location page, with its + * campaign tags + */ + public static void write( + PrintStream output, + String examplePage, + String locationPage) { + output.println(); + output.println(RULE); + output.println(HEADING); + output.println("\tThis example: " + examplePage); + output.println("\tLocation: " + locationPage); + output.println("\t" + REPOSITORY); + output.println(RULE); + output.flush(); + } +} diff --git a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/ResourceKey.java b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/ResourceKey.java new file mode 100644 index 000000000..884473091 --- /dev/null +++ b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/ResourceKey.java @@ -0,0 +1,56 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation.examples; + +/** + * Finds the resource key a Location example runs with. + */ +public final class ResourceKey { + + /** + * The variable the examples read their resource key from. + */ + public static final String VARIABLE = "_51DEGREES_RESOURCE_KEY"; + + private ResourceKey() { + } + + /** + * The key given as the first argument, or else the key in + * {@link #VARIABLE}, read as an environment variable and then as a + * system property. + * @param args the example's arguments + * @return the key, or null when there is none + */ + public static String find(String[] args) { + for (String value : new String[]{ + args.length > 0 ? args[0] : null, + System.getenv(VARIABLE), + System.getProperty(VARIABLE)}) { + if (value != null && value.trim().isEmpty() == false) { + return value.trim(); + } + } + return null; + } +} diff --git a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java index 8c86dfe01..419d236dd 100644 --- a/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java +++ b/location/geo-location.shell.examples/src/main/java/fiftyone/geolocation/examples/cloud/GettingStarted.java @@ -25,6 +25,8 @@ import fiftyone.geolocation.GeoLocationPipelineBuilder; import fiftyone.geolocation.core.Enums; import fiftyone.geolocation.core.data.GeoData; +import fiftyone.geolocation.examples.FindOutMore; +import fiftyone.geolocation.examples.ResourceKey; import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.Pipeline; @@ -32,7 +34,6 @@ import org.slf4j.ILoggerFactory; import org.slf4j.LoggerFactory; -import java.util.concurrent.Future; import static fiftyone.geolocation.core.Constants.EVIDENCE_GEO_LAT_PARAM_KEY; import static fiftyone.geolocation.core.Constants.EVIDENCE_GEO_LON_PARAM_KEY; @@ -47,12 +48,10 @@ * Required Maven Dependencies: * - [com.51degrees.geo-location](https://central.sonatype.com/artifact/com.51degrees/geo-location/) * - * 1. Build a new Pipeline to use the 51Degrees cloud service, with lazy loading - * configured to allow up to a second for a response from the service. + * 1. Build a new Pipeline to use the 51Degrees cloud service. * ``` * Pipeline pipeline = new GeoLocationPipelineBuilder(loggerFactory) * .useCloud(resourceKey, Enums.GeoLocationProvider.FiftyOneDegrees) - * .useLazyLoading(1000) * .build(); * ``` * @@ -82,13 +81,18 @@ public class GettingStarted { private static ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory(); public static void main(String[] args) throws Exception { - // Obtain a resource key for free at https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=geo-location.shell.examples-src-main-java-fiftyone-geolocation-examples-cloud-gettingstarted.java&utm_term=main + // Obtain a resource key for free at https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=geo-location.shell.examples-src-main-java-fiftyone-geolocation-examples-cloud-gettingstarted.java&utm_term=main // Make sure to include the 'Country' property as it is used by this example. - String resourceKey = "!!Your resource license key!!"; + String resourceKey = ResourceKey.find(args); - if (resourceKey.startsWith("!!")) { - System.out.println("You need to create a resource key at " + - "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=geo-location.shell.examples-src-main-java-fiftyone-geolocation-examples-cloud-gettingstarted.java&utm_term=resource-key-required and paste it into this example."); + if (resourceKey == null) { + System.out.println("This example calls the 51Degrees cloud " + + "service, which needs a resource key, and none is set."); + System.out.println("Set the " + ResourceKey.VARIABLE + + " environment variable to a resource key, or give the key " + + "as the first argument, and run the example again."); + System.out.println("Create a resource key at " + + "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=geo-location.shell.examples-src-main-java-fiftyone-geolocation-examples-cloud-gettingstarted.java&utm_term=resource-key-required"); System.out.println("Make sure to include the 'Country' " + "property as it is used by this example."); } @@ -96,7 +100,6 @@ public static void main(String[] args) throws Exception { try (Pipeline pipeline = new GeoLocationPipelineBuilder(loggerFactory) .useCloud(resourceKey, Enums.GeoLocationProvider.FiftyOneDegrees) .setEndPoint("https://cloud.51degrees.com/api/v4") - .useLazyLoading(1000) .setAutoCloseElements(true) .build(); FlowData flowData = pipeline.createFlowData()) { @@ -106,10 +109,6 @@ public static void main(String[] args) throws Exception { AspectPropertyValue country = flowData.get(GeoData.class).getCountry(); - Future future = flowData.get(GeoData.class).getProcessFuture(); - System.out.print("Awaiting response"); - outputUntilCancelled(".", 1000, future); - System.out.println(flowData.get(CloudRequestData.class).getJsonResponse()); System.out.println("Country: " + country.toString()); @@ -117,16 +116,9 @@ public static void main(String[] args) throws Exception { System.out.println(flowData.get(GeoData.class).getRegion()); } } - } - - private static void outputUntilCancelled( - String text, - int intervalMs, - Future future) throws InterruptedException { - while (future.isDone() == false) { - System.out.print(text); - Thread.sleep(intervalMs); - } - future.cancel(true); + FindOutMore.write( + System.out, + "https://51degrees.com/documentation/_reverse_geocoding__examples__getting_started_51_degrees_cloud.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=geo-location.shell.examples-gettingstarted.java&utm_term=find-out-more", + "https://51degrees.com/location?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=geo-location.shell.examples-gettingstarted.java&utm_term=find-out-more"); } } diff --git a/location/geo-location.shell.examples/src/test/java/fiftyone/geolocation/examples/CloudExamplesWithoutKeyTests.java b/location/geo-location.shell.examples/src/test/java/fiftyone/geolocation/examples/CloudExamplesWithoutKeyTests.java new file mode 100644 index 000000000..7810e84e6 --- /dev/null +++ b/location/geo-location.shell.examples/src/test/java/fiftyone/geolocation/examples/CloudExamplesWithoutKeyTests.java @@ -0,0 +1,72 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation.examples; + +import fiftyone.geolocation.examples.cloud.GettingStarted; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Runs each Location example as it runs from a fresh clone, with no + * resource key anywhere, and checks that it ends without an error, names + * the variable to set and links to where a key is created, then ends with + * its Find out more section. + */ +public class CloudExamplesWithoutKeyTests { + + private static final String CONFIGURE = "https://51degrees.com/configure"; + + private static final String TAGS = + "?utm_source=code&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content="; + + private static void runWithoutKey(Class example) throws Exception { + ExampleProcess.Result result = ExampleProcess.runWithoutKeys(example); + System.out.println(result.output); + String output = result.output; + assertEquals(example.getSimpleName() + " should end with exit code 0", + 0, result.exitCode); + assertTrue("The output should name _51DEGREES_RESOURCE_KEY", + output.contains("Set the _51DEGREES_RESOURCE_KEY " + + "environment variable")); + int link = output.indexOf(CONFIGURE); + assertTrue("The output should link to where a key is created", + link >= 0 && output.startsWith(TAGS, + output.indexOf('?', link))); + assertTrue("The output should end with the Find out more section " + + "after the link", + output.lastIndexOf("Find out more") > link); + } + + @Test + public void gettingStarted() throws Exception { + runWithoutKey(GettingStarted.class); + } + + @Test + public void combiningServices() throws Exception { + runWithoutKey(CombiningServices.class); + } +} diff --git a/location/geo-location.shell.examples/src/test/java/fiftyone/geolocation/examples/ExampleProcess.java b/location/geo-location.shell.examples/src/test/java/fiftyone/geolocation/examples/ExampleProcess.java new file mode 100644 index 000000000..1fd9df08b --- /dev/null +++ b/location/geo-location.shell.examples/src/test/java/fiftyone/geolocation/examples/ExampleProcess.java @@ -0,0 +1,127 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation.examples; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +/** + * Runs an example's main method in a new Java process with no key in its + * environment, as someone runs it from a fresh clone, for the tests of + * what an example does with no settings. A new process is needed because + * a running Java process cannot remove a variable from its own + * environment. + */ +final class ExampleProcess { + + /** + * How long an example may run before the test stops it. + */ + private static final long TIMEOUT_SECONDS = 120; + + /** + * What an example wrote and how it ended. + */ + public static final class Result { + /** + * The process's exit code. + */ + public final int exitCode; + + /** + * Everything the process wrote to standard output and standard + * error. + */ + public final String output; + + Result(int exitCode, String output) { + this.exitCode = exitCode; + this.output = output; + } + } + + private ExampleProcess() { + } + + /** + * Runs the example's main method with no arguments and with every + * environment variable whose name holds "KEY", in any case, left out. + * The process takes the class path of the process that starts it, and + * of its system properties only project.root, which says where the + * files the example reads are. + * @param example the class whose main method runs + * @return the exit code and the output + * @throws Exception if the process cannot start, or does not end + * within the timeout + */ + public static Result runWithoutKeys(Class example) throws Exception { + List command = new ArrayList<>(); + command.add(new File(new File(System.getProperty("java.home"), + "bin"), "java").getPath()); + command.add("-cp"); + command.add(System.getProperty("java.class.path")); + String projectRoot = System.getProperty("project.root"); + if (projectRoot != null) { + command.add("-Dproject.root=" + projectRoot); + } + command.add(example.getName()); + + ProcessBuilder builder = new ProcessBuilder(command); + builder.environment().keySet().removeIf( + name -> name.toUpperCase(Locale.ROOT).contains("KEY")); + builder.redirectErrorStream(true); + Process process = builder.start(); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + Thread reader = new Thread(() -> copy(process.getInputStream(), + output)); + reader.start(); + if (process.waitFor(TIMEOUT_SECONDS, TimeUnit.SECONDS) == false) { + process.destroyForcibly(); + reader.join(); + throw new AssertionError(example.getName() + + " did not end within " + TIMEOUT_SECONDS + " seconds. " + + "It wrote:\n" + output.toString("UTF-8")); + } + reader.join(); + return new Result(process.exitValue(), output.toString("UTF-8")); + } + + private static void copy(InputStream input, ByteArrayOutputStream output) { + byte[] buffer = new byte[8192]; + try { + int read; + while ((read = input.read(buffer)) >= 0) { + output.write(buffer, 0, read); + } + } catch (IOException e) { + // The process ended, so there is no more to read. + } + } +} diff --git a/location/geo-location.shell.examples/src/test/java/fiftyone/geolocation/examples/FindOutMoreTests.java b/location/geo-location.shell.examples/src/test/java/fiftyone/geolocation/examples/FindOutMoreTests.java new file mode 100644 index 000000000..2cbaf72a1 --- /dev/null +++ b/location/geo-location.shell.examples/src/test/java/fiftyone/geolocation/examples/FindOutMoreTests.java @@ -0,0 +1,135 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation.examples; + +import fiftyone.geolocation.examples.cloud.GettingStarted; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Each Location example ends its output with a "Find out more" section, + * which lists pages on 51degrees.com, each with its campaign tags, and the + * repository. With no resource key set, each example says where to create + * one and stops, so these tests need no key. With a key set, as it is in + * CI, each example calls the cloud service and writes what it found before + * the section. + */ +public class FindOutMoreTests { + + private static final String RULE = + "----------------------------------------------------------------"; + + private static final String HEADING = "Find out more"; + + private static final String REPOSITORY = + "https://github.com/51Degrees/51degrees-java"; + + private static final String NO_KEY = + "needs a resource key, and none is set"; + + private static final Pattern TAGGED = Pattern.compile( + "https://51degrees\\.com/[^?\\s]*\\?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=[a-z0-9][a-z0-9._-]*&utm_term=find-out-more$"); + + /** + * Runs a Location example as a person would. + */ + private interface Example { + void run() throws Exception; + } + + @Test + public void GettingStarted_EndsWithFindOutMore() throws Exception { + assertFindOutMore(run(() -> GettingStarted.main(new String[0]))); + } + + @Test + public void CombiningServices_EndsWithFindOutMore() throws Exception { + assertFindOutMore(run(() -> CombiningServices.main(new String[0]))); + } + + private static void assertFindOutMore(String output) { + int heading = output.lastIndexOf(RULE + "\n" + HEADING + "\n"); + assertTrue("No Find out more section:\n" + output, heading >= 0); + String before = output.substring(0, heading); + if (ResourceKey.find(new String[0]) == null) { + assertTrue( + "The example did not say it needs a resource key before " + + "the Find out more section:\n" + output, + before.contains(NO_KEY)); + } else { + assertFalse( + "The example said no key is set, but " + + ResourceKey.VARIABLE + " is set:\n" + output, + before.contains(NO_KEY)); + assertFalse( + "The example wrote nothing before the Find out more " + + "section:\n" + output, + before.trim().isEmpty()); + } + String section = output.substring( + heading + RULE.length() + HEADING.length() + 2); + int end = section.indexOf(RULE + "\n"); + assertTrue("The section has no closing rule:\n" + output, end > 0); + section = section.substring(0, end); + int pages = 0; + for (String line : section.split("\n")) { + if (line.contains("51degrees.com")) { + assertTrue( + "A 51degrees.com link without its campaign tags: " + + line, + TAGGED.matcher(line).find()); + pages++; + } + } + assertEquals("The pages listed:\n" + section, 2, pages); + assertTrue("The repository is not listed:\n" + section, + section.contains(REPOSITORY)); + } + + /** + * Runs the example and returns what it wrote, with each line ending in + * a line feed. + */ + private static String run(Example example) throws Exception { + PrintStream original = System.out; + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + System.setOut(new PrintStream(captured, true, "UTF-8")); + try { + example.run(); + } finally { + System.setOut(original); + } + return new String(captured.toByteArray(), StandardCharsets.UTF_8) + .replace("\r\n", "\n"); + } +} diff --git a/location/geo-location.web.examples/geo-location.web.examples.mvc/src/main/java/fiftyone/geolocation/web/examples/mvc/ExampleLauncher.java b/location/geo-location.web.examples/geo-location.web.examples.mvc/src/main/java/fiftyone/geolocation/web/examples/mvc/ExampleLauncher.java index ebbc00c97..61263567b 100644 --- a/location/geo-location.web.examples/geo-location.web.examples.mvc/src/main/java/fiftyone/geolocation/web/examples/mvc/ExampleLauncher.java +++ b/location/geo-location.web.examples/geo-location.web.examples.mvc/src/main/java/fiftyone/geolocation/web/examples/mvc/ExampleLauncher.java @@ -31,6 +31,6 @@ public static void main(String[] args) throws LifecycleException { runWebApp( "geo-location.web.examples/geo-location.web.examples.mvc/src/main/webapp", "geo-location.web.examples/geo-location.web.examples.mvc/target", - 8080); + 5151); } } diff --git a/location/geo-location.web.examples/geo-location.web.examples.mvc/src/main/java/fiftyone/geolocation/web/examples/mvc/configuration/ExampleMvcConfig.java b/location/geo-location.web.examples/geo-location.web.examples.mvc/src/main/java/fiftyone/geolocation/web/examples/mvc/configuration/ExampleMvcConfig.java index 96503b92d..4229c9dbb 100644 --- a/location/geo-location.web.examples/geo-location.web.examples.mvc/src/main/java/fiftyone/geolocation/web/examples/mvc/configuration/ExampleMvcConfig.java +++ b/location/geo-location.web.examples/geo-location.web.examples.mvc/src/main/java/fiftyone/geolocation/web/examples/mvc/configuration/ExampleMvcConfig.java @@ -82,6 +82,9 @@ * ... * @RequestMapping(method = RequestMethod.GET) * public String get(ModelMap model, HttpServletRequest request) { + * // A Flow Data cannot be used by two threads at once. Work handed to + * // another thread or task gets the values it needs, read here on the + * // request's thread, not the Flow Data. * FlowData data = flowDataProvider.getFlowData(request); * GeoData geo = data.get(GeoData.class); * diff --git a/location/geo-location.web.examples/geo-location.web.examples.mvc/src/main/java/fiftyone/geolocation/web/examples/mvc/controller/ExampleController.java b/location/geo-location.web.examples/geo-location.web.examples.mvc/src/main/java/fiftyone/geolocation/web/examples/mvc/controller/ExampleController.java index 1a2a09d29..9b92046a9 100644 --- a/location/geo-location.web.examples/geo-location.web.examples.mvc/src/main/java/fiftyone/geolocation/web/examples/mvc/controller/ExampleController.java +++ b/location/geo-location.web.examples/geo-location.web.examples.mvc/src/main/java/fiftyone/geolocation/web/examples/mvc/controller/ExampleController.java @@ -46,6 +46,9 @@ public ExampleController(FlowDataProvider flowDataProvider) { @RequestMapping(method = RequestMethod.GET) public String get(ModelMap model, HttpServletRequest request) { + // A Flow Data cannot be used by two threads at once. Work handed to + // another thread or task gets the values it needs, read here on the + // request's thread, not the Flow Data. FlowData data = flowDataProvider.getFlowData(request); GeoData geo = data.get(GeoData.class); diff --git a/location/geo-location.web.examples/geo-location.web.examples.mvc/src/main/webapp/WEB-INF/51Degrees-Cloud.xml b/location/geo-location.web.examples/geo-location.web.examples.mvc/src/main/webapp/WEB-INF/51Degrees-Cloud.xml index 533cb4db2..db68881b1 100644 --- a/location/geo-location.web.examples/geo-location.web.examples.mvc/src/main/webapp/WEB-INF/51Degrees-Cloud.xml +++ b/location/geo-location.web.examples/geo-location.web.examples.mvc/src/main/webapp/WEB-INF/51Degrees-Cloud.xml @@ -27,7 +27,7 @@ https://cloud.51degrees.com/api/v4 diff --git a/location/geo-location.web.examples/geo-location.web.examples.servlet/src/main/java/fiftyone/geolocation/web/examples/servlet/Example.java b/location/geo-location.web.examples/geo-location.web.examples.servlet/src/main/java/fiftyone/geolocation/web/examples/servlet/Example.java index 6899aff99..5452e232f 100644 --- a/location/geo-location.web.examples/geo-location.web.examples.servlet/src/main/java/fiftyone/geolocation/web/examples/servlet/Example.java +++ b/location/geo-location.web.examples/geo-location.web.examples.servlet/src/main/java/fiftyone/geolocation/web/examples/servlet/Example.java @@ -77,7 +77,7 @@ public class Example extends HttpServlet { public static void main(String[] args) throws LifecycleException { EmbedTomcat.runWebApp("geo-location.web.examples/geo-location.web.examples.servlet/src/main/webapp", "geo-location.web.examples/geo-location.web.examples.servlet/target", - 8080); + 5151); } /** * Serializable class version number, which is used during deserialization. @@ -96,6 +96,9 @@ public static void main(String[] args) throws LifecycleException { */ protected void processRequest(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException, NoValueException { + // A Flow Data cannot be used by two threads at once. Work handed to + // another thread or task gets the values it needs, read here on the + // request's thread, not the Flow Data. FlowData data = flowDataProvider.getFlowData(request); GeoData geo = data.get(GeoData.class); response.setContentType("text/html;charset=UTF-8"); diff --git a/location/geo-location.web.examples/geo-location.web.examples.servlet/src/main/webapp/WEB-INF/51Degrees-Cloud.xml b/location/geo-location.web.examples/geo-location.web.examples.servlet/src/main/webapp/WEB-INF/51Degrees-Cloud.xml index 23e052eee..5a3b9582f 100644 --- a/location/geo-location.web.examples/geo-location.web.examples.servlet/src/main/webapp/WEB-INF/51Degrees-Cloud.xml +++ b/location/geo-location.web.examples/geo-location.web.examples.servlet/src/main/webapp/WEB-INF/51Degrees-Cloud.xml @@ -27,7 +27,7 @@ https://cloud.51degrees.com/api/v4 diff --git a/location/geo-location/src/main/java/fiftyone/geolocation/GeoLocationCloudPipelineBuilder.java b/location/geo-location/src/main/java/fiftyone/geolocation/GeoLocationCloudPipelineBuilder.java index 4cece057e..0c2e328bc 100644 --- a/location/geo-location/src/main/java/fiftyone/geolocation/GeoLocationCloudPipelineBuilder.java +++ b/location/geo-location/src/main/java/fiftyone/geolocation/GeoLocationCloudPipelineBuilder.java @@ -75,6 +75,10 @@ public Pipeline build() throws Exception { if (licenseKey != null && licenseKey.isEmpty() == false) { cloudRequestEngineBuilder.setLicenseKey(licenseKey); } + if (requestedProperties.isEmpty() == false) { + cloudRequestEngineBuilder.setRequestedProperties( + requestedProperties); + } if (cloudRequestOrigin != null && cloudRequestOrigin.isEmpty() == false) { cloudRequestEngineBuilder.setCloudRequestOrigin(cloudRequestOrigin); } diff --git a/location/geo-location/src/main/java/fiftyone/geolocation/GeoLocationPipelineBuilder.java b/location/geo-location/src/main/java/fiftyone/geolocation/GeoLocationPipelineBuilder.java index 464d12a4d..ecb3d61a2 100644 --- a/location/geo-location/src/main/java/fiftyone/geolocation/GeoLocationPipelineBuilder.java +++ b/location/geo-location/src/main/java/fiftyone/geolocation/GeoLocationPipelineBuilder.java @@ -63,7 +63,7 @@ public GeoLocationPipelineBuilder( * Use the 51Degrees cloud service to perform Geo Location. * @param resourceKey the resource key to use when querying the * service. - * Obtain one from https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=geo-location-src-main-java-fiftyone-geolocation-geolocationpipelinebuilder.java&utm_term=usecloud + * Obtain one from https://51degrees.com/configure?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=geo-location-src-main-java-fiftyone-geolocation-geolocationpipelinebuilder.java&utm_term=usecloud * @param provider the Geo Location provider to use * @return builder that can be used to configure and build a pipeline * that will use the cloud Geo Location engine @@ -79,7 +79,7 @@ public GeoLocationCloudPipelineBuilder useCloud( /** * Use the 51Degrees cloud service to perform Geo Location * @param resourceKey the resource to use when querying the cloud service. - * Obtain one from https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=geo-location-src-main-java-fiftyone-geolocation-geolocationpipelinebuilder.java&utm_term=usecloud-2 + * Obtain one from https://51degrees.com/configure?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=geo-location-src-main-java-fiftyone-geolocation-geolocationpipelinebuilder.java&utm_term=usecloud-2 * @param endpoint the 51Degrees cloud URL * @param provider the Geo Location provider to use * @return a builder that can be used to configure and build a pipeline that diff --git a/location/geo-location/src/test/java/fiftyone/geolocation/Constants.java b/location/geo-location/src/test/java/fiftyone/geolocation/Constants.java index 451015994..dc142cb8a 100644 --- a/location/geo-location/src/test/java/fiftyone/geolocation/Constants.java +++ b/location/geo-location/src/test/java/fiftyone/geolocation/Constants.java @@ -25,7 +25,10 @@ public class Constants { public static final String TEST_LAT = "51.457714"; public static final String TEST_LON = "-0.975973"; - public static final String RESOURCE_KEY_ENV_VAR = "SuperResourceKey"; + /** + * The key with every property, which the value tests need. + */ + public static final String RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY_SUPER"; // For some reason, the value of the 'county' property returned // by the call to the cloud is always null on the build agent, // despite working correctly locally. diff --git a/location/geo-location/src/test/java/fiftyone/geolocation/ElementDataKeyFormatTests.java b/location/geo-location/src/test/java/fiftyone/geolocation/ElementDataKeyFormatTests.java new file mode 100644 index 000000000..6a57f74d8 --- /dev/null +++ b/location/geo-location/src/test/java/fiftyone/geolocation/ElementDataKeyFormatTests.java @@ -0,0 +1,75 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation; + +import fiftyone.geolocation.core.Enums; +import fiftyone.geolocation.flowelements.GeoLocationCloudEngine; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.util.Locale; + +import static org.junit.Assert.assertTrue; + +/** + * The string key of the Location cloud engine's data contains only + * alphanumeric characters, full stop or hyphen. + *

    + * The key for the DigitalElement provider, "location_digitalelement", is + * left out. It is the name the cloud service gives that product in its + * response and property list, which the engine reads by its key, and the + * name every language uses, so it cannot change here alone. + *

    + * Specification: pipeline-specification/features/access-to-results.md line 7 + * at 51Degrees/specifications 706de1f0, "Strings SHOULD only contain + * alphanumeric characters, full stop or hyphen." + */ +public class ElementDataKeyFormatTests { + + private static String key(Enums.GeoLocationProvider provider) { + return new GeoLocationCloudEngine( + LoggerFactory.getLogger(GeoLocationCloudEngine.class), + null, + provider).getElementDataKey(); + } + + @Test + public void keys_OnlyAlphanumericFullStopOrHyphen() { + String key = key(Enums.GeoLocationProvider.FiftyOneDegrees); + assertTrue(key, key.matches("[A-Za-z0-9.-]+")); + } + + /** + * Every provider's key, the DigitalElement one included, is lower-case. + * Specification: access-to-results.md line 10, "Lower-case SHOULD be + * used." + */ + @Test + public void keys_LowerCase() { + for (Enums.GeoLocationProvider provider : + Enums.GeoLocationProvider.values()) { + String key = key(provider); + assertTrue(key, key.equals(key.toLowerCase(Locale.ROOT))); + } + } +} diff --git a/location/geo-location/src/test/java/fiftyone/geolocation/GeoLocationCloudPipelineBuilderTests.java b/location/geo-location/src/test/java/fiftyone/geolocation/GeoLocationCloudPipelineBuilderTests.java new file mode 100644 index 000000000..2c2943105 --- /dev/null +++ b/location/geo-location/src/test/java/fiftyone/geolocation/GeoLocationCloudPipelineBuilderTests.java @@ -0,0 +1,252 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation; + +import fiftyone.geolocation.core.Enums; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.services.HttpClient; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * The cloud Pipeline builder passes the keys and the requested properties + * to the Cloud Request Engine, so a Pipeline can authenticate on a license + * key alone. A fake cloud service records each data request. + */ +public class GeoLocationCloudPipelineBuilderTests { + + private static final String BASE = "https://cloud.example/api/v4/"; + private static final String LICENSE_KEY = "abcdefghijkl1234"; + private static final String RESOURCE_KEY = "resourcekey"; + private static final String FIRST = "location.country"; + private static final String SECOND = "location.town"; + private static final String PROPERTIES_RESPONSE = + "{\"Products\":{\"location\":{\"DataTier\":\"t\",\"Properties\":" + + "[{\"Name\":\"Country\",\"Type\":\"String\"," + + "\"Category\":\"c\"}]}}}"; + private static final String DATA_RESPONSE = "{\"location\":{}}"; + + /** Records every data request and answers from fixed responses. */ + private static class FakeCloud implements HttpClient { + final List dataRequests = + Collections.synchronizedList(new ArrayList()); + + @Override + public HttpURLConnection connect(URL url) { + return new HttpURLConnection(url) { + @Override + public void connect() { + } + + @Override + public void disconnect() { + } + + @Override + public boolean usingProxy() { + return false; + } + + @Override + public int getResponseCode() { + return 200; + } + }; + } + + @Override + public String postData( + HttpURLConnection connection, + Map headers, + byte[] data) { + return answer( + connection.getURL(), + data == null ? "" : + new String(data, StandardCharsets.UTF_8)); + } + + @Override + public String getResponseString(HttpURLConnection connection) { + return answer(connection.getURL(), ""); + } + + @Override + public String getResponseString( + HttpURLConnection connection, + Map headers) { + return answer(connection.getURL(), ""); + } + + private String answer(URL url, String body) { + String path = url.getPath(); + if (path.endsWith("accessibleproperties")) { + return PROPERTIES_RESPONSE; + } + if (path.endsWith("evidencekeys")) { + return "[\"query.51D_Pos_latitude\"]"; + } + dataRequests.add(body); + return DATA_RESPONSE; + } + } + + private FakeCloud cloud; + + @Before + public void setUp() { + cloud = new FakeCloud(); + } + + private GeoLocationCloudPipelineBuilder builder(String resourceKey) { + return new GeoLocationPipelineBuilder( + LoggerFactory.getILoggerFactory(), cloud) + .useCloud( + resourceKey, + Enums.GeoLocationProvider.FiftyOneDegrees) + .setEndPoint(BASE); + } + + /** Process one request and give the fields of the form it sent. */ + private Map process(GeoLocationCloudPipelineBuilder builder) + throws Exception { + try (Pipeline pipeline = builder.build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.51D_Pos_latitude", "51.5").process(); + } + assertEquals(1, cloud.dataRequests.size()); + return form(cloud.dataRequests.get(0)); + } + + private static Map form(String body) + throws UnsupportedEncodingException { + Map fields = new HashMap<>(); + for (String pair : body.split("&")) { + if (pair.isEmpty()) { + continue; + } + String[] parts = pair.split("=", 2); + fields.put( + URLDecoder.decode(parts[0], "UTF-8"), + URLDecoder.decode(parts.length > 1 ? parts[1] : "", "UTF-8")); + } + return fields; + } + + @Test + public void LicenseKeyAlone_RequestedProperties_SentAsValues() + throws Exception { + Map sent = process(builder(null) + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Arrays.asList(FIRST, SECOND))); + + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST + "," + SECOND, sent.get("values")); + assertFalse(sent.containsKey("resource")); + } + + @Test + public void LicenseKeyAlone_RequestedPropertiesAsText_SentAsValues() + throws Exception { + Map sent = process(builder("") + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(FIRST + ", " + SECOND)); + + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST + "," + SECOND, sent.get("values")); + assertFalse(sent.containsKey("resource")); + } + + @Test + public void BothKeys_RequestedProperties_SendsAllThree() + throws Exception { + Map sent = process(builder(RESOURCE_KEY) + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Collections.singletonList(FIRST))); + + assertEquals(RESOURCE_KEY, sent.get("resource")); + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST, sent.get("values")); + } + + @Test + public void ResourceKey_NoRequestedProperties_SendsNoValues() + throws Exception { + Map sent = process(builder(RESOURCE_KEY)); + + assertEquals(RESOURCE_KEY, sent.get("resource")); + assertFalse(sent.containsKey("license")); + assertFalse(sent.containsKey("values")); + } + + @Test + public void LicenseKeyAlone_NoRequestedProperties_Refused() + throws Exception { + try { + builder(null).setLicenseKey(LICENSE_KEY).build().close(); + fail("A license key with no requested properties was built."); + } catch (PipelineConfigurationException e) { + assertTrue(e.getMessage(), + e.getMessage().contains("answers 400")); + assertTrue(e.getMessage(), + e.getMessage().contains("setRequestedProperties")); + assertFalse(e.getMessage().contains(LICENSE_KEY)); + } + assertTrue(cloud.dataRequests.isEmpty()); + } + + @Test + public void SetRequestedProperties_Null_Refused() { + try { + builder(RESOURCE_KEY).setRequestedProperties((List) null); + fail("A null list was accepted."); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("properties")); + } + try { + builder(RESOURCE_KEY).setRequestedProperties((String) null); + fail("A null string was accepted."); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("properties")); + } + } +} diff --git a/location/geo-location/src/test/java/fiftyone/geolocation/TimedHttpClient.java b/location/geo-location/src/test/java/fiftyone/geolocation/TimedHttpClient.java new file mode 100644 index 000000000..ed8575ec3 --- /dev/null +++ b/location/geo-location/src/test/java/fiftyone/geolocation/TimedHttpClient.java @@ -0,0 +1,81 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation; + +import fiftyone.pipeline.engines.services.HttpClientDefault; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.util.Map; + +/** + * Sends requests as the default client does, and prints how long each one + * to the live cloud service took, so each run records the measurement the + * tests' cloud timeout rests on. Only the path is printed, never the query, + * and a path's last part ending in ".json" is replaced, as it is the + * Resource Key. + */ +class TimedHttpClient extends HttpClientDefault { + + @Override + public String postData( + HttpURLConnection connection, + Map headers, + byte[] data) throws IOException { + long start = System.currentTimeMillis(); + String outcome = "failed"; + try { + String reply = super.postData(connection, headers, data); + outcome = "answered"; + return reply; + } finally { + print(connection, outcome, start); + } + } + + @Override + public String getResponseString( + HttpURLConnection connection, + Map headers) throws IOException { + long start = System.currentTimeMillis(); + String outcome = "failed"; + try { + String reply = super.getResponseString(connection, headers); + outcome = "answered"; + return reply; + } finally { + print(connection, outcome, start); + } + } + + private static void print( + HttpURLConnection connection, + String outcome, + long start) { + System.out.println( + "Cloud request " + + connection.getURL().getPath() + .replaceAll("[^/]+\\.json$", ".json") + " " + + outcome + " in " + (System.currentTimeMillis() - start) + " ms"); + } +} diff --git a/location/geo-location/src/test/java/fiftyone/geolocation/Wrapper51Degrees.java b/location/geo-location/src/test/java/fiftyone/geolocation/Wrapper51Degrees.java index 579dab11d..669e87ad7 100644 --- a/location/geo-location/src/test/java/fiftyone/geolocation/Wrapper51Degrees.java +++ b/location/geo-location/src/test/java/fiftyone/geolocation/Wrapper51Degrees.java @@ -33,6 +33,8 @@ import org.slf4j.LoggerFactory; +import static org.junit.Assume.assumeFalse; + public class Wrapper51Degrees implements Wrapper { protected static final TestLoggerFactory logger = @@ -48,18 +50,30 @@ public Wrapper51Degrees() throws Exception { String resourceKey = null; if(envResourceKey == null || envResourceKey.isEmpty()) { - if (propertyResourceKey == null || propertyResourceKey.isEmpty()) - throw new Exception("Resource key is required to run Cloud tests."); - else { - resourceKey = propertyResourceKey; - } + assumeFalse("These tests call the live 51Degrees cloud service, so " + + "they need a resource key, and they are skipped because " + + Constants.RESOURCE_KEY_ENV_VAR + " is not set as an " + + "environment variable or as a system property.", + propertyResourceKey == null || propertyResourceKey.isEmpty()); + resourceKey = propertyResourceKey; } else { resourceKey = envResourceKey; } - cloudRequestEngine = new CloudRequestEngineBuilder(logger) + cloudRequestEngine = new CloudRequestEngineBuilder(logger, new TimedHttpClient()) .setResourceKey(resourceKey) + // The live cloud service is shared with every CI run. In run + // 44881631156 its slowest answer was 0.77 s, over 540 + // requests on all 15 Build and Test jobs (TimedHttpClient + // prints each one). In java #190's release build (run + // 44702767584, about 08:38 UTC on 20 November 2026) device + // detection's ValueCloudTests got "Read timed out" past the + // 2 second default on two jobs. How long those answers took + // was not measured. 15 seconds is about 20 times the slowest + // answer measured. These tests check what the service + // answers, not how fast. + .setTimeOutSeconds(15) .build(); engine = new GeoLocationCloudEngineBuilder(logger) .setGeoLocationProvider("FiftyOneDegrees") diff --git a/location/pom.xml b/location/pom.xml index 9974783aa..2c2b7cf2d 100644 --- a/location/pom.xml +++ b/location/pom.xml @@ -63,7 +63,7 @@ 4.5.130 4.13.2 - 1.7.36 + 2.0.20 1.3.16 4.3.1 20211205 @@ -78,7 +78,7 @@ 3.0.0 3.0.1 0.7.0 - 3.26.0 + 3.28.0 true false @@ -192,12 +192,10 @@ ${maven.multiModuleProjectDirectory} - - ${SuperResourceKey} - ${AcceptChNoneKey} - ${AcceptChPlatformKey} - ${AcceptChBrowserKey} - ${AcceptChHardwareKey} + + <_51DEGREES_RESOURCE_KEY_SUPER>${_51DEGREES_RESOURCE_KEY_SUPER} @@ -344,9 +342,11 @@ https://opensource.org/licenses/EUPL-1.2 - - https://github.com/51Degrees/location-java - scm:git:git://github.com/51Degrees/location-java.git - scm:git:info@51degrees.com:51Degrees/location-java.git + + https://github.com/51Degrees/51degrees-java + scm:git:https://github.com/51Degrees/51degrees-java.git + scm:git:https://github.com/51Degrees/51degrees-java.git diff --git a/pipeline/README.md b/pipeline/README.md index 66b7d319d..81ee1bc1e 100644 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -28,6 +28,7 @@ but additional caution should be applied. - pipeline.engines - Shared functionality that is available to all 'engines' (specialized 'flow elements') - pipeline.engines.fiftyone - Functionality that is specific to 51Degrees engines. - pipeline.cloudrequestengine - An engine that is used to retrieve data from 51Degrees' cloud API. +- pipeline.derivedproperty - The Derived Property Element, which produces new properties from the properties other elements produce, following rules written in a script. Every Derived Property Element in a Pipeline writes into the same Element Data, so two of them must not be members of the same parallel group. ## Installation @@ -113,6 +114,42 @@ This affects `SequenceElementBuilder`, `SetHeadersElementBuilder` and `ShareUsag which ship inside `pipeline.engines.fiftyone`. Short names continue to work for every builder on the class path. +## Web integration + +`pipeline.web` provides `PipelineFilter`, a servlet filter, and +`pipeline.web.mvc` provides `FiftyOneInterceptor`, a Spring MVC interceptor. +Each creates one Flow Data for each request, processes it, and closes it when +the request ends. Application code reads it through `FlowDataProviderCore` or +`FlowDataProvider`. + +### Flow Data and threads + +A Flow Data cannot be used by two threads at once. A servlet or a controller +handles each request on one thread, so reading the Flow Data there is safe. An +application that hands work from a request to another thread or task reads the +values that work needs from the Flow Data on the request's thread, into its +own variables, and passes those values, not the Flow Data. The Flow Data is +closed when the request ends, so work that outlives the request must not keep +it either. A Flow Data is not safe to use from several threads at once, and an +application that shares one across threads is responsible for any locking. + +This controller, based on the +[clientside-element-mvc](pipeline.developer-examples/pipeline.developer-examples.clientside-element-mvc) +example, hands a value to an executor. + +```java +@RequestMapping(method = RequestMethod.GET) +public String get(ModelMap model, HttpServletRequest request) { + FlowData data = flowDataProvider.getFlowData(request); + // Read on the request's thread, into a variable of its own. + String starSign = data.get(StarSignData.class).getStarSign(); + // Hand the value, not the Flow Data, to the other thread. + executor.submit(() -> statistics.count(starSign)); + model.addAttribute("starSign", starSign); + return "example"; +} +``` + ## Tests Most packages contain tests which use junit and mockito. These tests can be run using maven by calling: `mvn test` @@ -139,7 +176,7 @@ and on our [website](https://51degrees.com/documentation/_device_detection__exam Every example that calls the 51Degrees cloud takes its endpoint from the `_51DEGREES_CLOUD_ENDPOINT` environment variable, which is the cloud API base including the `/api/v4/` segment and defaults to `https://cloud.51degrees.com/api/v4/` (the cloud-engine example defaults to the star sign service it was written for, as its row above says). This is the same variable the cloud request engine in this repository honours, so setting it once points every example, and any pipeline built with the engine, at the same place. The resource key comes from `_51DEGREES_RESOURCE_KEY`. -The variable was called `FOD_CLOUD_API_URL` before, and that name is deprecated. The cloud request engine, the 51Did client and the cloud-engine example still read `FOD_CLOUD_API_URL` when `_51DEGREES_CLOUD_ENDPOINT` is not set, and each logs a warning the first time it does. When both are set, `_51DEGREES_CLOUD_ENDPOINT` is used. +The variable was called `FOD_CLOUD_API_URL` before. That name is no longer read, so set `_51DEGREES_CLOUD_ENDPOINT` in its place. A host other than cloud.51degrees.com would be used to (a) use an on premise web server, or (b) use a privately hosted version of the 51Degrees cloud for performance reasons. This is the private hosting option of the cloud service. Both run the same service, so the examples work unchanged. diff --git a/pipeline/derived-properties b/pipeline/derived-properties new file mode 160000 index 000000000..cc3063863 --- /dev/null +++ b/pipeline/derived-properties @@ -0,0 +1 @@ +Subproject commit cc30638632050795d9acc80d5118a290ffaba085 diff --git a/pipeline/pipeline.caching/src/main/java/fiftyone/caching/LruCacheBase.java b/pipeline/pipeline.caching/src/main/java/fiftyone/caching/LruCacheBase.java index df581768e..72d4006bd 100644 --- a/pipeline/pipeline.caching/src/main/java/fiftyone/caching/LruCacheBase.java +++ b/pipeline/pipeline.caching/src/main/java/fiftyone/caching/LruCacheBase.java @@ -165,8 +165,11 @@ protected CachedItem add(K key, V value) { } else { if (updateExisting) { - newNode.list.replace(node, newNode); - node = newNode; + // The replacement takes the existing item's place in the + // existing item's list, so it must belong to that list. + CachedItem replacement = new CachedItem(node.list, key, value); + node.list.replace(node, replacement); + node = replacement; } // The item is in the dictionary. // Move the item to the head of it's LRU list. diff --git a/pipeline/pipeline.caching/src/test/java/fiftyone/caching/LruPutCacheTests.java b/pipeline/pipeline.caching/src/test/java/fiftyone/caching/LruPutCacheTests.java index d7c469d4f..798b15c93 100644 --- a/pipeline/pipeline.caching/src/test/java/fiftyone/caching/LruPutCacheTests.java +++ b/pipeline/pipeline.caching/src/test/java/fiftyone/caching/LruPutCacheTests.java @@ -218,4 +218,72 @@ public void LruPutCache_Replace() { "The existing value was not overwritten in the cache", "replacement", result); } + + /** + * A replaced item stays in the linked list the item it replaces was in, + * so a cache with several lists stays whole through replacements and the + * evictions after them. Each new item is put on a list chosen at random, + * so with four lists and forty replacements, a replacement that joined + * another list is all but certain to break the lists checked here. + */ + @Test + public void LruPutCache_ReplaceWithSeveralLists() { + final int size = 10; + PutCache cache = null; + cache = new LruPutCache.Builder() + .setUpdateExisting(true) + .setConcurrency(4) + .build(cache, size); + for (int i = 0; i < size; i++) { + cache.put(i, "first " + i); + } + for (int round = 0; round < 4; round++) { + for (int i = 0; i < size; i++) { + cache.put(i, "replacement " + round + " " + i); + } + } + assertListsWhole((LruPutCache) cache); + for (int i = 0; i < size; i++) { + assertEquals("replacement 3 " + i, cache.get(i)); + } + + // Put past the cache's size, which removes items from the ends of + // the lists, and read the items back. + for (int i = size; i < size * 10; i++) { + cache.put(i, "value " + i); + cache.get(i); + } + assertListsWhole((LruPutCache) cache); + } + + /** + * More items than any list in LruPutCache_ReplaceWithSeveralLists can + * hold, so a walk that takes more steps has found a loop. + */ + private static final int MAX_LIST_STEPS = 1000; + + /** + * Each linked list runs from its first item to its last through items + * that belong to it, with each item's previous item the one before it. + */ + private static void assertListsWhole(LruPutCache cache) { + for (LruCacheBase.CacheLinkedList list : + cache.linkedLists) { + LruCacheBase.CachedItem previous = null; + LruCacheBase.CachedItem item = list.first; + int steps = 0; + while (item != null) { + assertTrue("A list loops back on itself", + ++steps <= MAX_LIST_STEPS); + assertSame("An item is in a list it does not belong to", + list, item.list); + assertSame("An item's previous item is not the one before it", + previous, item.previous); + previous = item; + item = item.next; + } + assertSame("The list's last item is not the end of its chain", + previous, list.last); + } + } } diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java index eebb82eb4..71080b387 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java @@ -32,7 +32,7 @@ public class Messages { + "to authenticate with, as the cloud service answers 401 to a " + "request that carries neither. Use setResourceKey(String) with " + "a resource key created for free at " - + "https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.cloudrequestengine-src-main-java-fiftyone-pipeline-cloudrequestengine-constants.java&utm_term=no-credential"; + + "https://51degrees.com/configure?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.cloudrequestengine-src-main-java-fiftyone-pipeline-cloudrequestengine-constants.java&utm_term=no-credential"; public static final String LicenseKeyNeedsProperties = "The Cloud Request Engine has a license key and no requested " @@ -58,6 +58,16 @@ public class Messages { public static final String EvidenceConflict = "'%s:%s' evidence conflicts with %s"; + + /** + * Warning a cloud engine logs when the Cloud Request Engine has no + * response for it to read. The argument is the engine's class name. + */ + public static final String NoCloudResponse = + "The %s did not process, as the Cloud Request Engine has no " + + "response. The Cloud Request Engine's error in the log says " + + "why."; + public static final String MessageNoDataInResponse = "No data in response from cloud service at '%s'"; public static final String MessageErrorCodeReturned = diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java index 80b39e9a6..51c2d99c0 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java @@ -23,6 +23,7 @@ package fiftyone.pipeline.cloudrequestengine.flowelements; import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.MessageProductNotInResourceKey; +import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.NoCloudResponse; import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.ProcessCloudEngineNotImplemented; import fiftyone.pipeline.cloudrequestengine.NotImplementedException; @@ -31,6 +32,7 @@ import fiftyone.pipeline.core.data.ElementPropertyMetaData; import fiftyone.pipeline.core.data.ElementPropertyMetaDataDefault; import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; import fiftyone.pipeline.core.data.factories.ElementDataFactory; import fiftyone.pipeline.core.data.types.JavaScript; import fiftyone.pipeline.core.flowelements.FlowElement; @@ -49,6 +51,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @@ -309,7 +312,7 @@ private Map buildMetaDataMap( List properties) { Map result = new HashMap<>(); for (ElementPropertyMetaData property : properties) { - result.put(property.getName().toLowerCase(), property); + result.put(property.getName().toLowerCase(Locale.ROOT), property); } return result; } @@ -346,9 +349,9 @@ protected Map createAPVMap( for (Map.Entry property : cloudData.entrySet()) { if (property.getKey().endsWith("nullreason") == false) { Object outputValue = property.getValue(); - if (metaDataMap.containsKey(property.getKey().toLowerCase())) { + if (metaDataMap.containsKey(property.getKey().toLowerCase(Locale.ROOT))) { ElementPropertyMetaData metaData = - metaDataMap.get(property.getKey().toLowerCase()); + metaDataMap.get(property.getKey().toLowerCase(Locale.ROOT)); // If this property has a type of AspectPropertyValue // then create a new instance and populate it. @@ -397,13 +400,14 @@ protected Map createAPVMap( protected void processEngine(FlowData data, TData aspectData) { CloudRequestData requestData; - + CloudRequestEngine requestEngine; + // Get requestData from CloudRequestEngine. If requestData does not // exist in the element data TypedKeyMap then the CloudRequestEngine either // does not exist in the Pipeline or is not run before this engine. try { - requestData = data.getFromElement( - getRequestEngine().getInstance()); + requestEngine = getRequestEngine().getInstance(); + requestData = data.getFromElement(requestEngine); } catch(Exception ex) { throw new PipelineConfigurationException( @@ -421,9 +425,10 @@ protected void processEngine(FlowData data, TData aspectData) { "before it in the Pipeline. This engine will be unable " + "to produce results until this is corrected."); } - - String json = requestData == null ? null : requestData.getJsonResponse(); - + + String json = + getJsonResponse(data, aspectData, requestData, requestEngine); + // If the JSON is empty or null then do not Process the CloudAspectEngine. // Empty or null JSON indicates that an error has occurred in the // CloudRequestEngine. The error will have been reported by the @@ -432,14 +437,112 @@ protected void processEngine(FlowData data, TData aspectData) { if (json != null && !json.isEmpty()) { processCloudEngine(data, aspectData, json); } else { - logger.warn("The " + this.getClass().getSimpleName() + " did not process " + - "as the JSON response from the CloudRequestEngine was null " + - "or empty. Please refer to errors generated by the " + - "CloudRequestEngine in the logs as this indicates an error " + - "occurred there."); + logger.warn(String.format( + NoCloudResponse, getClass().getSimpleName())); } } + /** + * The JSON response of the Cloud Request Engine's request for the Flow + * Data, waiting for the request to finish. A failed request met here, + * while the Flow Data is processed, is recorded once in the Flow Data's + * errors as the Cloud Request Engine's error, not this engine's, and the + * Element Data is marked so that a property it does not hold is missing + * with the CloudRequestFailed reason. An engine that overrides + * {@code processEngine} reads the response through this. + * @param data the Flow Data being processed + * @param aspectData the Element Data this engine populates + * @param requestData the Cloud Request Engine's Element Data + * @param requestEngine the Cloud Request Engine of the Pipeline + * @return the JSON response, or null where the request failed + */ + protected String getJsonResponse( + FlowData data, + TData aspectData, + CloudRequestData requestData, + CloudRequestEngine requestEngine) { + String json; + try { + json = requestData.getJsonResponse(); + } catch (RuntimeException e) { + // The request's own methods, which the public type does not show. + CloudRequestDataInternal request = requestData; + Throwable failure = request.getRequestFailure(); + if (failure == null) { + throw e; + } + if (request.firstToMeetFailure()) { + data.addError(failure, requestEngine); + } + json = null; + } + setCloudRequestFailed(data, aspectData, requestEngine); + return json; + } + + /** + * Where the Cloud Request Engine's request for the Flow Data failed, + * record it in the Element Data, so that a property the Element Data + * does not hold is missing with the CloudRequestFailed reason. The + * request itself says whether it failed, so the answer does not depend + * on whether another member of a parallel group has recorded the failure + * in the Flow Data's errors yet. A failure the Cloud Request Engine threw + * while it processed, before any request started, is in those errors. + * @param data the Flow Data being processed + * @param aspectData the Element Data this engine populates + * @param requestEngine the Cloud Request Engine of the Pipeline + */ + protected void setCloudRequestFailed( + FlowData data, + TData aspectData, + CloudRequestEngine requestEngine) { + if (aspectData instanceof AspectDataBase == false) { + return; + } + if (requestFailed(data, requestEngine)) { + ((AspectDataBase) aspectData).setCloudRequestFailed(); + } + } + + /** + * @return true if the Cloud Request Engine's request for the Flow Data + * failed, or the engine threw while it processed + */ + private static boolean requestFailed( + FlowData data, + CloudRequestEngine requestEngine) { + CloudRequestDataInternal request = data.getFromElement(requestEngine); + if (request != null && request.getRequestFailure() != null) { + return true; + } + if (data.getErrors() == null) { + return false; + } + for (FlowError error : data.getErrors()) { + if (error.getFlowElement() == requestEngine) { + return true; + } + } + return false; + } + + /** + * The request for the property list failed where the Cloud Request + * Engine still does not know the accessible properties, because it + * requests them when it is built and again on first use. + * @return true if the request for the property list failed + */ + @Override + public boolean propertyListRequestFailed() { + CloudRequestEngine requestEngine; + try { + requestEngine = getRequestEngine().getInstance(); + } catch (PipelineConfigurationException e) { + return false; + } + return requestEngine.getPublicProperties() == null; + } + /** * A virtual method to be implemented by the derived class which * uses the JsonResponse from the CloudRequestEngine to populate the TData diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestDataInternal.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestDataInternal.java index b3145b2b8..656eba700 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestDataInternal.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestDataInternal.java @@ -23,22 +23,149 @@ package fiftyone.pipeline.cloudrequestengine.flowelements; import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineDataException; import fiftyone.pipeline.engines.data.AspectData; import fiftyone.pipeline.engines.data.AspectDataBase; import fiftyone.pipeline.engines.data.AspectPropertyMetaData; import fiftyone.pipeline.engines.flowelements.AspectEngine; import org.slf4j.Logger; -public class CloudRequestDataInternal extends AspectDataBase { +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.atomic.AtomicBoolean; + +public class CloudRequestDataInternal extends AspectDataBase + implements AutoCloseable { private static final String JSON_RESPONSE_KEY = "json-response"; private static final String PROCESS_STARTED_KEY = "process-started"; + /** + * The cloud request, started while the engine processes. The JSON + * response is a lazy property whose value is this task's result, so the + * task never writes this Element Data. + */ + private volatile CompletableFuture jsonResponseTask = null; + + /** + * Set by the first reader to meet the task's failure, so the failure is + * recorded in the Flow Data's errors once. + */ + private final AtomicBoolean failureMet = new AtomicBoolean(false); + + /** + * The JSON response from the cloud. Where the engine started a request, + * this waits for it to finish, and every caller gets the same response + * or the same failure. + * @return the JSON response, or null if there is none + * @throws PipelineDataException if the cloud request failed, being the + * failure itself where it is unchecked, or wrapping it otherwise + */ public String getJsonResponse() { + CompletableFuture task = jsonResponseTask; + if (task != null) { + try { + return task.join(); + } catch (CompletionException e) { + Throwable cause = e.getCause() == null ? e : e.getCause(); + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + throw new PipelineDataException(cause.getMessage(), cause); + } + } if(super.get(JSON_RESPONSE_KEY) != null) return super.get(JSON_RESPONSE_KEY).toString(); return null; } + /** + * The JSON response asked for by name waits for the request too. + */ + @Override + public Object get(String propertyName) { + if (jsonResponseTask != null && + JSON_RESPONSE_KEY.equalsIgnoreCase(propertyName)) { + return getJsonResponse(); + } + return super.get(propertyName); + } + + /** + * Add the started cloud request as the task for the JSON response. + * Called by the engine while it processes. + * @param task the started request + */ + void setJsonResponseTask(CompletableFuture task) { + jsonResponseTask = task; + } + + /** + * The failure of the cloud request as it was thrown, waiting for the + * request to finish. + * @return the failure, or null if there is no request or it succeeded + */ + Throwable getRequestFailure() { + CompletableFuture task = jsonResponseTask; + if (task == null) { + return null; + } + try { + task.join(); + return null; + } catch (CompletionException e) { + return e.getCause() == null ? e : e.getCause(); + } + } + + /** + * @return true for the first caller only, which records the failure + */ + boolean firstToMeetFailure() { + return failureMet.compareAndSet(false, true); + } + + /** + * Wait for a cloud request still running, so closing the Flow Data does + * not leave it behind. Readers get any failure, so this ignores it. + */ + @Override + public void close() { + CompletableFuture task = jsonResponseTask; + if (task != null) { + try { + task.join(); + } catch (CompletionException e) { + // Left to the readers. + } + } + } + + /** + * The JSON response as it is now, without waiting for the cloud request, + * so the engine can check it before the result is cached or served from + * the cache. + * @return the JSON response, or null if there is none, or the cloud + * request is still running or failed + */ + String getJsonResponseWithoutWaiting() { + CompletableFuture task = jsonResponseTask; + if (task != null) { + return task.isDone() && task.isCompletedExceptionally() == false ? + task.join() : null; + } + Object response = getWithoutWaiting(JSON_RESPONSE_KEY); + return response == null ? null : response.toString(); + } + + /** + * Whether the cloud request the engine started is still running. + * @return true if a request was started and has not finished + */ + boolean isRequestRunning() { + CompletableFuture task = jsonResponseTask; + return task != null && task.isDone() == false; + } + public CloudRequestDataInternal( Logger logger, diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java index db9f39443..c458af103 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java @@ -23,6 +23,7 @@ package fiftyone.pipeline.cloudrequestengine.flowelements; +import fiftyone.caching.PutCacheBuilder; import fiftyone.pipeline.annotations.DefaultValue; import fiftyone.pipeline.annotations.ElementBuilder; import fiftyone.pipeline.cloudrequestengine.Constants.Messages; @@ -43,7 +44,6 @@ import java.util.List; import java.util.Set; import java.util.TreeSet; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; /** @@ -72,11 +72,6 @@ public class CloudRequestEngineBuilder extends return System.getenv(name); }; - // Set once the use of the deprecated end point variable has been - // logged, so that a process building many engines logs it once. - private static final AtomicBoolean oldEndPointVariableLogged = - new AtomicBoolean(false); - public CloudRequestEngineBuilder(ILoggerFactory loggerFactory) { this(loggerFactory, new HttpClientDefault()); } @@ -97,8 +92,7 @@ protected CloudRequestEngine newEngine(List properties) throws Exception /* * Check if endPoint has been explicitly set via setEndpoint. - * If not check for environment variable _51DEGREES_CLOUD_ENDPOINT, - * and then for its deprecated name FOD_CLOUD_API_URL. + * If not check for environment variable _51DEGREES_CLOUD_ENDPOINT. * If nothing else is set, use default value. */ if (endPoint == null || endPoint.isEmpty()) { @@ -145,6 +139,7 @@ protected CloudRequestEngine newEngine(List properties) throws Exception * @param state the copy, or null to discover when building * @return this builder */ + @DefaultValue("None - discovered when the engine is built") public CloudRequestEngineBuilder setDiscoveryState( CloudDiscoveryState state) { this.discoveryState = state; @@ -163,30 +158,11 @@ public CloudDiscoveryState getDiscoveryState() { } /** - * The end point the environment gives, or null or empty when it gives - * none. {@link Constants#CLOUD_ENDPOINT_VARIABLE} is read first. Only - * when that is not set or is empty is the deprecated - * {@link Constants#FOD_CLOUD_API_URL} read, and the first use of it in - * the process is logged as a warning. + * The end point that {@link Constants#CLOUD_ENDPOINT_VARIABLE} gives, + * or null or empty when it gives none. */ - @SuppressWarnings("deprecation") private String endPointFromEnvironment() { - String value = getEnvVar.apply(Constants.CLOUD_ENDPOINT_VARIABLE); - if (value != null && value.isEmpty() == false) { - return value; - } - value = getEnvVar.apply(Constants.FOD_CLOUD_API_URL); - if (value != null && value.isEmpty() == false - && oldEndPointVariableLogged.compareAndSet(false, true)) { - loggerFactory.getLogger(CloudRequestEngineBuilder.class.getName()) - .warn("The environment variable " - + Constants.FOD_CLOUD_API_URL + " is deprecated and " - + Constants.CLOUD_ENDPOINT_VARIABLE + " replaces it. " - + "Set " + Constants.CLOUD_ENDPOINT_VARIABLE - + " to the same value, as a later release may no " - + "longer read " + Constants.FOD_CLOUD_API_URL + "."); - } - return value; + return getEnvVar.apply(Constants.CLOUD_ENDPOINT_VARIABLE); } private static String trimmed(String value) { @@ -271,7 +247,7 @@ public CloudRequestEngineBuilder setEndpoint(String uri) { * @param uri data endpoint * @return this builder */ - @DefaultValue(Constants.END_POINT_DEFAULT + "{resourcekey}.json") + @DefaultValue(Constants.END_POINT_DEFAULT + "/{resourcekey}.json") public CloudRequestEngineBuilder setDataEndpoint(String uri) { dataEndpoint = uri; return this; @@ -285,7 +261,7 @@ public CloudRequestEngineBuilder setDataEndpoint(String uri) { * @param uri properties endpoint * @return this builder */ - @DefaultValue(Constants.END_POINT_DEFAULT + "accessibleproperties") + @DefaultValue(Constants.END_POINT_DEFAULT + "/accessibleproperties") public CloudRequestEngineBuilder setPropertiesEndpoint(String uri) { propertiesEndpoint = uri; return this; @@ -299,7 +275,7 @@ public CloudRequestEngineBuilder setPropertiesEndpoint(String uri) { * @param uri evidence keys endpoint * @return this builder */ - @DefaultValue(Constants.END_POINT_DEFAULT + "evidencekeys") + @DefaultValue(Constants.END_POINT_DEFAULT + "/evidencekeys") public CloudRequestEngineBuilder setEvidenceKeysEndpoint(String uri) { evidenceKeysEndpoint = uri; return this; @@ -446,4 +422,28 @@ public CloudRequestEngineBuilder setCacheSize(int size) { this.cacheConfig = new CacheConfiguration(size); return this; } + + /** + * Where the cache is built by a {@link PutCacheBuilder}, as it is by + * {@link #setCacheSize(int)} and by a {@link CacheConfiguration} made + * with a size, the cache is set to replace an entry when a result is put + * for its key. A result is stored while its cloud request is still + * running, and an entry whose request then fails, or answers with a + * 51Did, is never served, so the next result for that key must take its + * place. The builder in the {@link CacheConfiguration} is set, not + * copied, so a configuration shared with another engine's builder is set + * for that engine too. + * @param engine the engine to configure + * @throws Exception if the engine could not be configured + */ + @Override + protected void configureEngine(CloudRequestEngine engine) + throws Exception { + if (cacheConfig != null && + cacheConfig.getCacheBuilder() instanceof PutCacheBuilder) { + ((PutCacheBuilder) cacheConfig.getCacheBuilder()) + .setUpdateExisting(true); + } + super.configureEngine(engine); + } } diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index b95d7ec5d..ec10340ca 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -56,6 +56,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -148,6 +151,17 @@ public class CloudRequestEngineDefault */ private final Object discoveryLock = new Object(); + /** + * Runs the cloud requests started while processing, on daemon threads + * so an engine not closed does not keep the process alive. + */ + private final ExecutorService requests = + Executors.newCachedThreadPool(runnable -> { + Thread thread = new Thread(runnable, "cloud-request"); + thread.setDaemon(true); + return thread; + }); + public CloudRequestEngineDefault( Logger logger, ElementDataFactory aspectDataFactory, @@ -457,6 +471,67 @@ private void completeDiscovery() throws IOException { } } + /** + * A response whose JSON has a top-level member named fodid, in any case, + * holds a 51Did created for the one request, so it is never served from + * the cache, and the request goes to the cloud service instead. See the + * caching specification: + * "A cache MUST NOT store a result that holds such a value, and MUST NOT + * serve one that it finds." A response that cannot be read as a JSON + * object is not cached either, as it cannot be checked. + *

    + * The response is a lazy task that is usually still running when the + * result is offered to the cache, and this check never waits for it. A + * result whose request is still running is stored, because what its + * response holds is not known yet, and {@link #isServable} keeps it from + * being served unless its request finished with a response that may be + * served. A result whose request has finished is stored only if that + * response may be served. + * @param data the result to check + * @return false if the request has finished and its response may not be + * cached + */ + @Override + protected boolean isCacheable(CloudRequestData data) { + if (((CloudRequestDataInternal)data).isRequestRunning()) { + return true; + } + return isServable(data); + } + + /** + * A result found in the cache is served only once its cloud request has + * finished, without failing, with a response that has no top-level fodid + * member. Otherwise the request goes to the cloud service, so a request + * still running for another Flow Data, and one that failed, are never + * shared through the cache. + * @param data the result found in the cache + * @return true if the result may be served + */ + @Override + protected boolean isServable(CloudRequestData data) { + String response = + ((CloudRequestDataInternal)data).getJsonResponseWithoutWaiting(); + if (response == null) { + return false; + } + try { + for (String key : new JSONObject(response).keySet()) { + if (FODID.equalsIgnoreCase(key)) { + return false; + } + } + return true; + } catch (JSONException e) { + return false; + } + } + + /** + * The top-level member of a cloud response that holds a 51Did. + */ + private static final String FODID = "fodid"; + @Override protected void processEngine(FlowData data, CloudRequestData aspectData) throws IOException { ((CloudRequestDataInternal)aspectData).setProcessStarted(true); @@ -471,12 +546,29 @@ protected void processEngine(FlowData data, CloudRequestData aspectData) throws headers.put("Content-Type", "application/x-www-form-urlencoded"); headers.put("Content-Length", Integer.toString(content.length)); - String response = httpClient.postData(connection, headers, content); - - ((CloudRequestDataInternal)aspectData).setJsonResponse(response); + // The request is built from the Flow Data above. The task holds only + // the request, and readers of the JSON response wait for it. + ((CloudRequestDataInternal)aspectData).setJsonResponseTask( + CompletableFuture.supplyAsync( + () -> send(connection, headers, content), requests)); + } - validateResponse(response, connection); - reportPropertiesNotCovered(response); + /** + * Send a built request and check its response, as the task behind the + * JSON response. Never reads the Flow Data. + */ + private String send( + HttpURLConnection connection, + Map headers, + byte[] content) { + try { + String response = httpClient.postData(connection, headers, content); + validateResponse(response, connection); + reportPropertiesNotCovered(response); + return response; + } catch (IOException e) { + throw new CompletionException(e); + } } /** @@ -794,6 +886,15 @@ private boolean hasKeyPrefix(String itemKey, String prefix) { @Override protected void unmanagedResourcesCleanup() { + // Wait for the requests already started, which the connection + // timeouts bound, so none outlives the engine and every reader + // still gets its response. + requests.shutdown(); + try { + requests.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } private void setCommonHeaders(Map headers) { @@ -976,6 +1077,17 @@ private void validateResponse(String jsonResult, .map(Object::toString) .collect(Collectors.toList())); } + + // Each warning is logged, whether or not the response also + // holds errors. The cloud's warnings can name the license key + // it was sent. + JSONArray warnings = jObj.optJSONArray("warnings"); + if (warnings != null) { + for (Object warning : warnings.toList()) { + logger.warn(LicenseKeyMask.maskIn( + String.valueOf(warning), getLicenseKeys())); + } + } } // If there were no errors but there was also no other data diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java index 84b3e024e..59277ed52 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java @@ -29,8 +29,8 @@ public class Constants { // Environment variable name to set end point public static final String CLOUD_ENDPOINT_VARIABLE = "_51DEGREES_CLOUD_ENDPOINT"; /** - * The old name of the environment variable that sets the end point, - * read only when {@link #CLOUD_ENDPOINT_VARIABLE} is not set. + * The old name of the environment variable that sets the end point. + * It is no longer read. * @deprecated use {@link #CLOUD_ENDPOINT_VARIABLE}, and set the * environment variable it names in place of this one. */ diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/PropertyKeyedCloudEngineBase.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/PropertyKeyedCloudEngineBase.java index 7d8a19821..805178b73 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/PropertyKeyedCloudEngineBase.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/PropertyKeyedCloudEngineBase.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.cloudrequestengine.flowelements; +import fiftyone.pipeline.core.data.ElementDataBase; import fiftyone.pipeline.core.data.ElementPropertyMetaData; import fiftyone.pipeline.core.data.EvidenceKeyFilter; import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; @@ -100,6 +101,10 @@ protected void processCloudEngine(FlowData data, TData aspectData, String json) profile.populateFromMap(profileData); //device.SetNoValueReasons(nullReasons); + // A profile is written only here, so it is finished at once. + if (profile instanceof ElementDataBase) { + ((ElementDataBase) profile).finishWriting(); + } aspectData.addProfile(profile); } } diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/BuilderDefaultValueTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/BuilderDefaultValueTests.java new file mode 100644 index 000000000..bfde8d476 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/BuilderDefaultValueTests.java @@ -0,0 +1,68 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.pipeline.annotations.DefaultValue; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Every configuration setter of the Cloud Request Engine builder gives its + * default value beside it, with {@link DefaultValue}. See + * reference-implementation-notes.md line 79: + * "In all cases, there SHOULD be some mechanism to specify the default + * values alongside configuration property setters." + */ +public class BuilderDefaultValueTests { + + static List missingDefaults(Class... builders) { + List missing = new ArrayList<>(); + for (Class builder : builders) { + for (Method method : builder.getDeclaredMethods()) { + if (Modifier.isPublic(method.getModifiers()) && + method.getName().startsWith("set") && + method.isSynthetic() == false && + method.isBridge() == false && + method.getAnnotation(DefaultValue.class) == null) { + missing.add(builder.getSimpleName() + "#" + + method.getName() + + Arrays.toString(method.getParameterTypes())); + } + } + } + return missing; + } + + @Test + public void cloudRequestEngineBuilderSetters_GiveTheirDefaults() { + assertEquals(new ArrayList(), + missingDefaults(CloudRequestEngineBuilder.class)); + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java index 4f073192e..46177ac95 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java @@ -131,6 +131,23 @@ public void addProfile(TestData profile) { } } + /** + * A property keyed cloud engine that declares no Evidence of its own + * takes the base class's default, which accepts no Evidence. + */ + @Test + public void propertyKeyedDefault_AcceptsNoEvidence() { + EvidenceKeyFilter filter = new TestKeyedInstance().getEvidenceKeyFilter(); + for (String key : Arrays.asList( + "header.user-agent", + "query.client-ip", + "server.client-ip", + "cookie.51D_example")) { + assertFalse(filter.include(key), + "The default Evidence filter accepted '" + key + "'."); + } + } + private class TestKeyedInstance extends PropertyKeyedCloudEngineBase { public TestKeyedInstance() { diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilderTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilderTests.java index 8159049a0..6c137d4b6 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilderTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilderTests.java @@ -22,11 +22,9 @@ package fiftyone.pipeline.cloudrequestengine.flowelements; -import fiftyone.common.testhelpers.TestLogger; import fiftyone.pipeline.cloudrequestengine.CloudRequestException; import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; import fiftyone.pipeline.engines.services.HttpClientDefault; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.LoggerFactory; @@ -34,10 +32,6 @@ import java.io.IOException; import java.lang.reflect.Field; import java.net.MalformedURLException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import java.util.function.Function; import static org.junit.jupiter.api.Assertions.*; @@ -46,17 +40,10 @@ public class CloudRequestEngineBuilderTests extends CloudRequestEngineTestsBase private final String testResourceKey = "resource_key"; private final String testEndpoint = "https://testEndpoint/"; private final String testEnvVarEndpoint = "https://testEnvVarEndpoint/"; - private final String testOldEnvVarEndpoint = - "https://testOldEnvVarEndpoint/"; // The deprecated name of the end point variable, which is also the - // name of the constant that holds it. + // name of the constant that holds it. The variable is no longer read. private static final String OLD_ENDPOINT_VARIABLE = "FOD_CLOUD_API_URL"; - - // The old name is logged as deprecated once in a process, so what was - // logged is carried from one test of this class to the next. - private static final List deprecationsInEarlierTests = - new ArrayList<>(); public CloudRequestEngineBuilderTests() throws MalformedURLException { super(); @@ -67,10 +54,6 @@ public void setUp() throws IOException { configureMockedClient(); } - @AfterEach - public void countDeprecations() { - deprecationsInEarlierTests.addAll(deprecationsLogged()); - } @Test @SuppressWarnings("unused") @@ -145,118 +128,6 @@ public void CloudEndPoint_Default_Setting() throws Exception { Constants.END_POINT_DEFAULT + "/", getEndpointValue(builder)); } - /** - * Gives the builder an environment holding only the variables in the - * map, so that a test chooses which of the two names is set. - */ - private void setEnvironment( - CloudRequestEngineBuilder builder, - Map environment) throws Exception { - Field getEnvVarField = builder.getClass().getDeclaredField("getEnvVar"); - getEnvVarField.setAccessible(true); - getEnvVarField.set( - builder, - (Function)(name) -> environment.get(name)); - } - - /** - * A builder that finds the old name in the environment, and the new - * name only where a value is given for it. - */ - private CloudRequestEngineBuilder builderWithOldName( - String newNameValue) throws Exception { - Map environment = new HashMap<>(); - environment.put(OLD_ENDPOINT_VARIABLE, testOldEnvVarEndpoint); - if (newNameValue != null) { - environment.put(Constants.CLOUD_ENDPOINT_VARIABLE, newNameValue); - } - CloudRequestEngineBuilder builder = - new CloudRequestEngineBuilder(loggerFactory, httpClient); - setEnvironment(builder, environment); - return builder.setResourceKey(testResourceKey); - } - - /** - * The warnings this test's loggers hold that name the old variable. - */ - private List deprecationsLogged() { - List result = new ArrayList<>(); - for (TestLogger logger : loggerFactory.loggers) { - for (String warning : logger.warningsLogged) { - if (warning.contains(OLD_ENDPOINT_VARIABLE)) { - result.add(warning); - } - } - } - return result; - } - - @Test - public void CloudEndPoint_Old_Environment_Name() throws Exception { - CloudRequestEngineBuilder builder = builderWithOldName(null); - - builder.build(); - - assertEquals(testOldEnvVarEndpoint, getEndpointValue(builder)); - assertEquals( - testOldEnvVarEndpoint + "accessibleproperties", - getPropertiesEndpointValue(builder)); - } - - @Test - public void CloudEndPoint_Old_Environment_Name_When_New_Is_Empty() - throws Exception { - CloudRequestEngineBuilder builder = builderWithOldName(""); - - builder.build(); - - assertEquals(testOldEnvVarEndpoint, getEndpointValue(builder)); - } - - @Test - public void CloudEndPoint_New_Environment_Name_Wins() throws Exception { - CloudRequestEngineBuilder builder = - builderWithOldName(testEnvVarEndpoint); - - builder.build(); - - assertEquals(testEnvVarEndpoint, getEndpointValue(builder)); - assertEquals( - testEnvVarEndpoint + "accessibleproperties", - getPropertiesEndpointValue(builder)); - assertEquals(0, deprecationsLogged().size()); - } - - @Test - public void CloudEndPoint_Explicit_Setting_Beats_Old_Name() - throws Exception { - CloudRequestEngineBuilder builder = builderWithOldName(null); - - builder.setEndpoint(testEndpoint).build(); - - assertEquals(testEndpoint, getEndpointValue(builder)); - assertEquals(0, deprecationsLogged().size()); - } - - /** - * Two engines built from the old name log one warning between them, - * however many of this class's tests read the old name before this one. - */ - @Test - public void CloudEndPoint_Old_Environment_Name_Logged_Once() - throws Exception { - builderWithOldName(null).build(); - builderWithOldName(null).build(); - - List logged = new ArrayList<>(deprecationsInEarlierTests); - logged.addAll(deprecationsLogged()); - assertEquals(1, logged.size(), logged.toString()); - String warning = logged.get(0); - assertTrue( - warning.contains(Constants.CLOUD_ENDPOINT_VARIABLE), warning); - assertTrue(warning.contains("deprecated"), warning); - } - @Test public void CloudEndPoint_Old_Constant_Is_Kept() throws Exception { Field old = Constants.class.getField(OLD_ENDPOINT_VARIABLE); diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineCacheKeyCaseTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineCacheKeyCaseTests.java new file mode 100644 index 000000000..2f001f8a0 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineCacheKeyCaseTests.java @@ -0,0 +1,121 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import org.json.JSONArray; +import org.junit.jupiter.api.Test; + +import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * An engine with a cache treats two requests whose Evidence differs only + * in the case of a key as the same request, and two whose Evidence differs + * only in the case of a value as different ones. See the Generation of keys + * section of + * caching.md: + * "Comparison of Evidence keys MUST be case-insensitive." + */ +public class CloudRequestEngineCacheKeyCaseTests + extends CloudRequestEngineTestsBase { + + public CloudRequestEngineCacheKeyCaseTests() throws MalformedURLException { + super(); + } + + /** + * Process the two pieces of Evidence in turn with one engine that has a + * cache, and count the requests that reach the cloud service. + */ + private int requestsFor(String[] first, String[] second) throws Exception { + evidenceKeysResponse = + new JSONArray(Arrays.asList("query.User-Agent")).toString(); + configureMockedClient(); + final List answers = new ArrayList<>(Arrays.asList( + "{'device':{'value':'1'}}", + "{'device':{'value':'2'}}")); + doAnswer(invocation -> { + java.net.HttpURLConnection connection = invocation.getArgument(0); + String path = connection.getURL().getPath(); + if (path.endsWith("properties")) { + return accessiblePropertiesResponse; + } + if (path.endsWith("evidencekeys")) { + return evidenceKeysResponse; + } + return answers.remove(0); + }).when(httpClient).postData(any(), anyMap(), (byte[]) any()); + + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .setCacheSize(10) + .build(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build()) { + for (String[] evidence : new String[][] { first, second }) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidence[0], evidence[1]); + data.process(); + data.get(CloudRequestData.class).getJsonResponse(); + } + } + } + return 2 - answers.size(); + } + + @Test + public void keyInAnotherCase_IsServedFromTheCache() throws Exception { + assertEquals(1, requestsFor( + new String[] { "query.User-Agent", "iPhone" }, + new String[] { "query.user-agent", "iPhone" }), + "A key in another case was not taken as the same Evidence."); + verify(httpClient, times(1)).postData( + argThat(c -> c.getURL().getPath().endsWith("properties") == false && + c.getURL().getPath().endsWith("evidencekeys") == false), + any(), + any()); + } + + @Test + public void valueInAnotherCase_IsNotServedFromTheCache() throws Exception { + assertEquals(2, requestsFor( + new String[] { "query.User-Agent", "iPhone" }, + new String[] { "query.User-Agent", "IPHONE" }), + "A value in another case was taken as the same Evidence."); + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineCloudErrorsTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineCloudErrorsTests.java new file mode 100644 index 000000000..27ec17c8c --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineCloudErrorsTests.java @@ -0,0 +1,115 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.pipeline.cloudrequestengine.CloudRequestException; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.exceptions.AggregateException; +import org.junit.jupiter.api.Test; + +import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.ExceptionCloudErrorsMultiple; +import static org.junit.jupiter.api.Assertions.*; + +/** + * The entries of the errors array in a response from the cloud service are + * thrown, one entry as a CloudRequestException carrying its text, and + * several as one AggregateException holding a CloudRequestException for + * each. + */ +public class CloudRequestEngineCloudErrorsTests + extends CloudRequestEngineTestsBase { + + public CloudRequestEngineCloudErrorsTests() throws MalformedURLException { + super(); + } + + @Test + public void Process_SeveralErrors_OneAggregateException() + throws Exception { + jsonResponse = "{\"errors\":[\"first error\",\"second error\"]}"; + configureMockedClient(); + + Throwable thrown = processOnce(); + + assertInstanceOf(AggregateException.class, thrown); + assertEquals(ExceptionCloudErrorsMultiple, thrown.getMessage()); + List messages = new ArrayList<>(); + for (Throwable inner : thrown.getSuppressed()) { + assertInstanceOf(CloudRequestException.class, inner); + assertEquals( + 200, + ((CloudRequestException) inner).getHttpStatusCode()); + messages.add(inner.getMessage()); + } + List expected = new ArrayList<>(); + expected.add("first error"); + expected.add("second error"); + assertEquals(expected, messages); + } + + @Test + public void Process_OneError_CloudRequestException() throws Exception { + jsonResponse = "{\"errors\":[\"only error\"]}"; + configureMockedClient(); + + Throwable thrown = processOnce(); + + assertInstanceOf(CloudRequestException.class, thrown); + assertTrue( + thrown.getMessage().contains("only error"), + thrown.getMessage()); + assertEquals(0, thrown.getSuppressed().length); + } + + /** + * Process one request, and give the one error it raised. + */ + private Throwable processOnce() throws Exception { + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(true) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + // The response is a lazy property, so once processing has + // finished its error reaches only the reader. + Collection errors = data.getErrors(); + assertEquals(0, errors.size(), errors.toString()); + return assertThrows(RuntimeException.class, + () -> data.getFromElement(engine).getJsonResponse()); + } + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineFodIdCacheTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineFodIdCacheTests.java new file mode 100644 index 000000000..8d17244a5 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineFodIdCacheTests.java @@ -0,0 +1,176 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.caching.FlowCache; +import org.json.JSONArray; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +/** + * A response that holds a value created for the one request that asked for + * it, such as a 51Did, is neither stored in nor served from the cache. + * See the Values created for one request section of + * caching.md: + * "A cache MUST NOT store a result that holds such a value, and MUST NOT + * serve one that it finds." + */ +public class CloudRequestEngineFodIdCacheTests + extends CloudRequestEngineTestsBase { + + public CloudRequestEngineFodIdCacheTests() throws MalformedURLException { + super(); + } + + /** + * Process the same Evidence twice with an engine that has a cache, the + * service answering each request with the next response in the list. + * @param responses the responses, one a request + * @return the JSON each request's Flow Data holds, in order + */ + private List processTwice(String... responses) throws Exception { + evidenceKeysResponse = + new JSONArray(Arrays.asList("query.User-Agent")).toString(); + configureMockedClient(); + final List remaining = new ArrayList<>(Arrays.asList(responses)); + doAnswer(invocation -> { + java.net.HttpURLConnection connection = invocation.getArgument(0); + String path = connection.getURL().getPath(); + if (path.endsWith("properties")) { + return accessiblePropertiesResponse; + } + if (path.endsWith("evidencekeys")) { + return evidenceKeysResponse; + } + return remaining.remove(0); + }).when(httpClient).postData(any(), anyMap(), (byte[]) any()); + + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .setCacheSize(10) + .build(); + List answers = new ArrayList<>(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build()) { + for (int i = 0; i < 2; i++) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + answers.add(data.get(CloudRequestData.class) + .getJsonResponse()); + } + } + } + return answers; + } + + /** + * A response whose JSON has a top-level member named fodid, in any case, + * is not stored, so the second identical request goes to the cloud + * service and gets its own 51Did. + */ + @ParameterizedTest + @ValueSource(strings = { "fodid", "FodId", "FODID" }) + public void process_FodIdResponseIsNotStored(String name) throws Exception { + String first = "{'device':{'value':'1'},'" + name + "':'first'}"; + String second = "{'device':{'value':'1'},'" + name + "':'second'}"; + + List answers = processTwice(first, second); + + assertEquals(Arrays.asList(first, second), answers, + "A response with a 51Did was served from the cache."); + verify(httpClient, times(2)).postData( + argThat(c -> c.getURL().getPath().endsWith("properties") == false), + any(), + any()); + } + + /** + * Every other response is cached as before, including one whose fodid + * member is not at the top level. + */ + @Test + public void process_ResponseWithoutTopLevelFodIdIsCached() + throws Exception { + String first = "{'device':{'value':'1','fodid':'nested'}}"; + + List answers = processTwice(first); + + assertEquals(Arrays.asList(first, first), answers); + verify(httpClient, times(1)).postData( + argThat(c -> c.getURL().getPath().endsWith("properties") == false), + any(), + any()); + } + + /** + * A result with a 51Did that the cache already holds, such as one an + * earlier version stored, is not served. The request goes to the cloud + * service instead. + */ + @Test + public void process_FodIdResultFoundInTheCacheIsNotServed() + throws Exception { + evidenceKeysResponse = + new JSONArray(Arrays.asList("query.User-Agent")).toString(); + configureMockedClient(); + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .build(); + FlowCache cache = mock(FlowCache.class); + engine.setCache(cache); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build(); + FlowData earlier = pipeline.createFlowData(); + FlowData data = pipeline.createFlowData()) { + CloudRequestData cached = new CloudRequestData( + loggerFactory.getLogger("test"), earlier, engine); + cached.setJsonResponse( + "{'device':{'value':'1'},'fodid':'stale'}"); + when(cache.get(any())).thenReturn(cached); + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + assertEquals(jsonResponse, + data.get(CloudRequestData.class).getJsonResponse(), + "A result with a 51Did found in the cache was served."); + verify(cache, never()).put(any(), same(cached)); + } + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLazyCacheTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLazyCacheTests.java new file mode 100644 index 000000000..28bc715d3 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLazyCacheTests.java @@ -0,0 +1,493 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.caching.Cache; +import fiftyone.caching.CacheBuilder; +import fiftyone.caching.LruPutCache; +import fiftyone.caching.PutCache; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.configuration.CacheConfiguration; +import fiftyone.pipeline.engines.configuration.LazyLoadingConfiguration; +import org.json.JSONArray; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +/** + * A Cloud Request Engine with a cache returns from process without waiting + * for the cloud service, because its request is a lazy task, and serves a + * result from the cache only once its request has finished with a response + * that may be served. See + * cloud-request-engine.md + * ("It then starts the HTTP request, adds its Element Data with + * json-response as a lazy property whose task is that request, and returns + * without waiting for the response.") and + * caching.md + * ("A cache MUST NOT store a result that holds such a value, and MUST NOT + * serve one that it finds."). The deprecated lazy loading option is set in + * these tests and has no effect. + *

    + * No result depends on timing: the cloud service's answer is held back until + * the test releases it, and the test reads the processing thread's stack to + * see whether it is waiting for that answer. + */ +public class CloudRequestEngineLazyCacheTests + extends CloudRequestEngineTestsBase { + + /** + * The timeout given with the lazy loading option, which has no effect. + */ + private static final int LAZY_TIMEOUT_MILLIS = 600000; + + public CloudRequestEngineLazyCacheTests() throws MalformedURLException { + super(); + } + + /** + * Mock the cloud service so that the nth data request gets the nth + * response in the list, and waits for the gate first. + */ + private void mockService( + final CountDownLatch called, + final CountDownLatch gate, + String... responses) throws Exception { + evidenceKeysResponse = + new JSONArray(Arrays.asList("query.User-Agent")).toString(); + configureMockedClient(); + final AtomicInteger next = new AtomicInteger(); + doAnswer(invocation -> { + HttpURLConnection connection = invocation.getArgument(0); + String path = connection.getURL().getPath(); + if (path.endsWith("properties")) { + return accessiblePropertiesResponse; + } + if (path.endsWith("evidencekeys")) { + return evidenceKeysResponse; + } + String response = responses[next.getAndIncrement()]; + called.countDown(); + gate.await(); + if (response == null) { + throw new IOException("The cloud service did not answer."); + } + return response; + }).when(httpClient).postData(any(), anyMap(), (byte[]) any()); + } + + private CloudRequestEngine buildLazyCachedEngine() throws Exception { + return new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .setCacheSize(10) + .setLazyLoading(new LazyLoadingConfiguration(LAZY_TIMEOUT_MILLIS)) + .build(); + } + + /** + * Whether the thread is waiting for a cloud request or a process task. + */ + private static boolean waitingForTheRequest(Thread thread) { + for (StackTraceElement frame : thread.getStackTrace()) { + String type = frame.getClassName(); + String method = frame.getMethodName(); + if (method.equals("waitOnAllProcessFutures") || + type.equals("java.util.concurrent.CompletableFuture") && + (method.equals("join") || method.equals("get") || + method.equals("waitingGet") || + method.equals("timedGet")) || + type.endsWith("FutureTask") && method.equals("get")) { + return true; + } + } + return false; + } + + /** + * process returns while the cloud service has not answered, so the + * check made before the result is stored does not wait for the + * response. + */ + @Test + public void process_CachedEngineDoesNotWaitForTheService() + throws Exception { + CountDownLatch called = new CountDownLatch(1); + CountDownLatch gate = new CountDownLatch(1); + mockService(called, gate, jsonResponse); + CloudRequestEngine engine = buildLazyCachedEngine(); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + final AtomicBoolean returned = new AtomicBoolean(false); + final AtomicReference failure = new AtomicReference<>(); + Thread processing = new Thread(() -> { + try { + data.process(); + returned.set(true); + } catch (Throwable t) { + failure.set(t); + } + }); + processing.start(); + called.await(); + // Look until process has returned, or is seen waiting for the + // answer that is held back. + boolean waited = false; + while (returned.get() == false && failure.get() == null && + processing.isAlive()) { + if (waitingForTheRequest(processing)) { + waited = true; + break; + } + Thread.yield(); + } + gate.countDown(); + processing.join(); + assertNull(failure.get()); + assertFalse(waited, + "process waited for the cloud service's answer."); + assertTrue(returned.get()); + assertEquals(jsonResponse, + data.get(CloudRequestData.class).getJsonResponse()); + } + } + + /** + * A failed request is not served from the cache, so the next identical + * request goes to the cloud service and gets its response. + */ + @Test + public void process_FailedRequestIsNotServed() throws Exception { + String second = "{'device':{'value':'1'}}"; + mockService(new CountDownLatch(2), new CountDownLatch(0), + null, second); + CloudRequestEngine engine = buildLazyCachedEngine(); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build()) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + CloudRequestData result = data.get(CloudRequestData.class); + assertThrows(RuntimeException.class, result::getJsonResponse); + } + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + assertEquals(second, + data.get(CloudRequestData.class).getJsonResponse(), + "The failed request was served from the cache."); + } + } + verify(httpClient, times(2)).postData( + argThat(c -> c.getURL().getPath().endsWith("properties") == false && + c.getURL().getPath().endsWith("evidencekeys") == false), + any(), + any()); + } + + /** + * A request still running for one Flow Data is not served from the cache + * to another, so the second gets its own request and its own response, + * which matters when the response holds a 51Did. + */ + @Test + public void process_RunningRequestIsNotShared() throws Exception { + String first = "{'device':{'value':'1'},'fodid':'first'}"; + String second = "{'device':{'value':'1'},'fodid':'second'}"; + CountDownLatch firstCalled = new CountDownLatch(1); + CountDownLatch gate = new CountDownLatch(1); + mockService(firstCalled, gate, first, second); + CloudRequestEngine engine = buildLazyCachedEngine(); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build(); + FlowData one = pipeline.createFlowData(); + FlowData two = pipeline.createFlowData()) { + one.addEvidence("query.User-Agent", "iPhone"); + one.process(); + // The first request has started and is held back, so its result + // is in the cache with its request still running. + firstCalled.await(); + two.addEvidence("query.User-Agent", "iPhone"); + two.process(); + gate.countDown(); + assertEquals(first, + one.get(CloudRequestData.class).getJsonResponse()); + assertEquals(second, + two.get(CloudRequestData.class).getJsonResponse(), + "A request still running was served from the cache."); + } + } + /** + * With lazy loading, a response without a 51Did is still cached, so the + * second identical request is served from the cache. + */ + @Test + public void process_LazyResponseWithoutFodIdIsCached() throws Exception { + CountDownLatch gate = new CountDownLatch(0); + String first = "{'device':{'value':'1'}}"; + mockService(new CountDownLatch(1), gate, first); + CloudRequestEngine engine = buildLazyCachedEngine(); + + List answers = processTwice(engine); + + assertEquals(Arrays.asList(first, first), answers); + verify(httpClient, times(1)).postData( + argThat(c -> c.getURL().getPath().endsWith("properties") == false && + c.getURL().getPath().endsWith("evidencekeys") == false), + any(), + any()); + } + + /** + * With lazy loading, a response with a 51Did is not served from the + * cache, so the second identical request goes to the cloud service and + * gets its own. + */ + @Test + public void process_LazyFodIdResponseIsNotServed() throws Exception { + CountDownLatch gate = new CountDownLatch(0); + String first = "{'device':{'value':'1'},'fodid':'first'}"; + String second = "{'device':{'value':'1'},'fodid':'second'}"; + mockService(new CountDownLatch(2), gate, first, second); + CloudRequestEngine engine = buildLazyCachedEngine(); + + List answers = processTwice(engine); + + assertEquals(Arrays.asList(first, second), answers, + "A response with a 51Did was served from the cache."); + } + + /** + * With lazy loading, the result is offered to the cache once, on the + * thread that called process. + */ + @Test + public void process_LazyResultIsStoredOnTheProcessingThread() + throws Exception { + mockService(new CountDownLatch(1), new CountDownLatch(0), + jsonResponse); + final List putThreads = + Collections.synchronizedList(new ArrayList<>()); + CacheBuilder recording = new CacheBuilder() { + @Override + public Cache build(Cache c, int cacheSize) { + final PutCache inner = + new LruPutCache.Builder().build(c, cacheSize); + return new PutCache() { + @Override + public void put(K key, V value) { + putThreads.add(Thread.currentThread()); + inner.put(key, value); + } + + @Override + public V get(K key) { + return inner.get(key); + } + + @Override + public void close() throws java.io.IOException { + inner.close(); + } + }; + } + }; + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .setCache(new CacheConfiguration(recording, 10)) + .setLazyLoading( + new LazyLoadingConfiguration(LAZY_TIMEOUT_MILLIS)) + .build(); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + assertEquals(jsonResponse, + data.get(CloudRequestData.class).getJsonResponse()); + } + + assertEquals(1, putThreads.size(), + "The result was not offered to the cache once."); + assertSame(Thread.currentThread(), putThreads.get(0), + "The processing task read the Flow Data to cache the result."); + } + + /** + * After a failed request, the next result for the same Evidence takes + * the failed entry's place, so the request after that is served from the + * cache. + */ + @Test + public void process_ResultAfterAFailureIsCached() throws Exception { + String second = "{'device':{'value':'1'}}"; + CountDownLatch gate = new CountDownLatch(1); + mockService(new CountDownLatch(2), gate, null, second); + CloudRequestEngine engine = buildLazyCachedEngine(); + + assertEquals(Arrays.asList(FAILED, second, second), + processThreeTimes(engine, gate)); + verifyDataRequests(2); + } + + /** + * After a response with a 51Did, the next result for the same Evidence + * takes its place, so the request after that is served from the cache. + * The cache is configured as the cloud pipeline builders configure it. + */ + @Test + public void process_ResultAfterAFodIdIsCached() throws Exception { + String first = "{'device':{'value':'1'},'fodid':'first'}"; + String second = "{'device':{'value':'1'}}"; + CountDownLatch gate = new CountDownLatch(1); + mockService(new CountDownLatch(2), gate, first, second); + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .setCache(new CacheConfiguration(10)) + .build(); + + assertEquals(Arrays.asList(first, second, second), + processThreeTimes(engine, gate)); + verifyDataRequests(2); + } + + /** + * After a failed entry is replaced, requests with other Evidence fill + * the cache past its size, and every request still gets its own + * response. + */ + @Test + public void process_ReplacedEntryThenEvictions() throws Exception { + final int others = 40; + String[] responses = new String[others + 2]; + responses[0] = null; + responses[1] = "{'device':{'value':'replacement'}}"; + for (int i = 0; i < others; i++) { + responses[i + 2] = "{'device':{'value':'" + i + "'}}"; + } + CountDownLatch gate = new CountDownLatch(1); + mockService(new CountDownLatch(1), gate, responses); + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .setCacheSize(4) + .build(); + + List answers = processThreeTimes(engine, gate); + assertEquals(Arrays.asList(FAILED, responses[1], responses[1]), + answers); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build()) { + for (int i = 0; i < others; i++) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "other " + i); + data.process(); + assertEquals(responses[i + 2], + data.get(CloudRequestData.class).getJsonResponse()); + } + } + } + } + + /** + * What processThreeTimes records for a request whose response could not + * be read. + */ + private static final String FAILED = "failed"; + + /** + * Process three Flow Data with the same Evidence and read each response. + * The first request is held back until its process has returned, so its + * result is offered to the cache while the request is still running. + */ + private List processThreeTimes( + CloudRequestEngine engine, + CountDownLatch firstGate) throws Exception { + List answers = new ArrayList<>(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build()) { + for (int i = 0; i < 3; i++) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + firstGate.countDown(); + try { + answers.add(data.get(CloudRequestData.class) + .getJsonResponse()); + } catch (RuntimeException e) { + answers.add(FAILED); + } + } + } + } + return answers; + } + + private void verifyDataRequests(int count) throws Exception { + verify(httpClient, times(count)).postData( + argThat(c -> c.getURL().getPath().endsWith("properties") == false && + c.getURL().getPath().endsWith("evidencekeys") == false), + any(), + any()); + } + + private List processTwice(CloudRequestEngine engine) + throws Exception { + List answers = new ArrayList<>(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build()) { + for (int i = 0; i < 2; i++) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + answers.add(data.get(CloudRequestData.class) + .getJsonResponse()); + } + } + } + return answers; + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLicenseKeyTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLicenseKeyTests.java index e05e9bc09..35715eea0 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLicenseKeyTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLicenseKeyTests.java @@ -46,6 +46,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import static org.junit.jupiter.api.Assertions.*; @@ -199,6 +200,15 @@ private String process(CloudRequestEngine engine) throws Exception { errors.append(t).append('\n'); } } + // The response is a lazy property, so once processing has + // finished its error reaches only the reader. + try { + data.getFromElement(engine).getJsonResponse(); + } catch (RuntimeException e) { + for (Throwable t = e; t != null; t = t.getCause()) { + errors.append(t).append('\n'); + } + } return errors.toString(); } } @@ -470,6 +480,46 @@ public void Discovery_LicenseKeyAlone_PostsCredentialsInTheBody() assertNull(evidence.get(0).url.getQuery()); } + /** + * The property metadata is the accessibleproperties answer, never one + * made from the requested properties. A requested name the answer does + * not list has none, a product the answer lists but no name requested + * is kept, and each type is the answer's. + */ + @Test + public void Discovery_LicenseKeyAlone_MetadataIsTheAnswerNotTheRequest() + throws Exception { + cloud.propertiesResponse = + "{\"Products\":{" + + "\"device\":{\"DataTier\":\"t\",\"Properties\":" + + "[{\"Name\":\"IsMobile\",\"Type\":\"Boolean\"," + + "\"Category\":\"Device\"}]}," + + "\"location\":{\"DataTier\":\"t\",\"Properties\":" + + "[{\"Name\":\"Town\",\"Type\":\"String\"," + + "\"Category\":\"Location\"}]}}}"; + CloudRequestEngine engine = build(builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Arrays.asList( + "device.ismobile", + "device.iscrawler", + "device.hardwarevendor"))); + + Map products = + engine.getPublicProperties(); + assertEquals( + new TreeSet<>(Arrays.asList("device", "location")), + new TreeSet<>(products.keySet())); + List device = + products.get("device").properties; + assertEquals(1, device.size()); + assertEquals("IsMobile", device.get(0).name); + assertEquals(boolean.class, device.get(0).getPropertyType()); + List location = + products.get("location").properties; + assertEquals(1, location.size()); + assertEquals("Town", location.get(0).name); + } + @Test public void Discovery_ResourceKeyAndProperties_GetsWithTheKeyInTheQuery() throws Exception { @@ -637,4 +687,51 @@ public void Process_ResourceKey_NothingToCompare() throws Exception { verify(logger, never()).warn(anyString()); } + + // The warnings array of the response, which the specification's HTTP + // requests section says is logged as warnings, as the errors array is + // thrown. + + @Test + public void Process_Warnings_EachLoggedAsAWarning() throws Exception { + cloud.dataResponse = "{\"device\":{\"ismobile\":true}," + + "\"warnings\":[\"First warning\",\"Second warning\"]}"; + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY)); + + assertEquals("", process(engine)); + + verify(logger, times(1)).warn("First warning"); + verify(logger, times(1)).warn("Second warning"); + } + + @Test + public void Process_WarningNamingTheLicenseKey_LoggedMasked() + throws Exception { + String warning = "License key " + LICENSE_KEY + " expires soon"; + cloud.dataResponse = "{\"device\":{\"ismobile\":true," + + "\"iscrawler\":false},\"warnings\":[\"" + warning + "\"]}"; + CloudRequestEngine engine = build(builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(PROPERTY_LIST)); + + assertEquals("", process(engine)); + + verify(logger, times(1)).warn(argThat((String m) -> + m.startsWith("License key ") && m.endsWith(" expires soon"))); + verify(logger, never()).warn(argThat((String m) -> + m.contains(LICENSE_KEY))); + } + + @Test + public void Process_NoWarnings_NothingLogged() throws Exception { + cloud.dataResponse = "{\"device\":{\"ismobile\":true}," + + "\"warnings\":[]}"; + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY)); + + assertEquals("", process(engine)); + + verify(logger, never()).warn(anyString()); + } } diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java index 8f8487633..46d44f5d4 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java @@ -30,14 +30,26 @@ import fiftyone.pipeline.engines.services.HttpClientDefault; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.TreeMap; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** * Calls the live 51Degrees cloud service through the cloud request @@ -65,14 +77,18 @@ public class CloudRequestEngineLiveTests { "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"; + /** A property every device data file has. */ + private static final String IS_MOBILE = "device.ismobile"; + + /** A name no product has, which the service leaves out of its answer. */ + private static final String NOT_A_PROPERTY = "device.notaproperty51d"; + @Test public void build_Process_ReadsDeviceDataFromTheLiveService() throws Exception { String resourceKey = LiveCloudKeys.find(LiveCloudKeys.RESOURCE_KEY_NAMES); - if (resourceKey == null) { - fail(LiveCloudKeys.noResourceKey()); - } + assumeTrue(resourceKey != null, LiveCloudKeys.noResourceKey()); CloudRequestEngineBuilder builder = new CloudRequestEngineBuilder( LoggerFactory.getILoggerFactory(), new HttpClientDefault()) @@ -98,7 +114,7 @@ public void build_Process_ReadsDeviceDataFromTheLiveService() "The live cloud service listed no products for this resource " + "key. A key carries the products its account holds, so this " + "is an entitlement problem and not a fault in the service. " - + "Create a key at https://configure.51degrees.com."); + + "Create a key at https://51degrees.com/configure."); assertTrue(products.containsKey("device"), "The live cloud service listed the products " + String.join(", ", products.keySet()) + " for this resource " @@ -136,4 +152,142 @@ public void build_Process_ReadsDeviceDataFromTheLiveService() } } } + + /** + * Authenticates on a license key alone, with no resource key, and + * names the properties wanted. The service then answers only those + * properties. One the license does not grant comes back as null with a + * reason, and a name that is not a property is left out without the + * service saying so, which the engine has to report itself, once. + *

    + * Skipped when no license key is set, or the one set grants no cloud + * product, because a resource key cannot stand in for one. No message + * here holds the service's answer, only the names of its fields, so + * that nothing the service says about the key reaches a build log. + */ + @Test + public void build_Process_LicenseKeyAlone_AnswersOnlyWhatWasAsked() + throws Exception { + String licenceKey = + LiveCloudKeys.find(LiveCloudKeys.LICENCE_KEY_NAMES); + assumeTrue(licenceKey != null, + String.join(", ", LiveCloudKeys.LICENCE_KEY_NAMES) + + " is not set, so there is no license key to authenticate with."); + + // The service answers a request that names no property the license + // grants with an error, so one granted property is asked for too. + // Discovery under the license key lists what the license grants. + String granted; + try (CloudRequestEngine discovery = new CloudRequestEngineBuilder( + LoggerFactory.getILoggerFactory(), new HttpClientDefault()) + .setLicenseKey(licenceKey) + .setRequestedProperties(Collections.singletonList(IS_MOBILE)) + .build()) { + granted = firstProperty(discovery.getPublicProperties()); + } + assumeTrue(granted != null, + "The live cloud service lists no product for this license " + + "key, so it is not a key for the cloud service."); + + Logger logger = mock(Logger.class); + ILoggerFactory loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + List asked = + Arrays.asList(granted, IS_MOBILE, NOT_A_PROPERTY); + CloudRequestEngine engine = new CloudRequestEngineBuilder( + loggerFactory, new HttpClientDefault()) + .setLicenseKey(licenceKey) + .setRequestedProperties(asked) + .build(); + + JSONObject answer = null; + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build()) { + // Twice, because the engine reports a name left out once and + // not on every request. + for (int i = 0; i < 2; i++) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", USER_AGENT) + .process(); + answer = new JSONObject( + data.getFromElement(engine).getJsonResponse()); + } + } + } + + assertFalse(answer.has("errors"), + "The live cloud service answered a license key and a list of " + + "properties with errors. Its answer has the fields " + + answer.keySet() + "."); + JSONObject device = answer.optJSONObject("device"); + assertTrue(device != null && device.has("ismobile"), + "The live cloud service left " + IS_MOBILE + " out of its " + + "answer to a license key that asked for it. Its answer has " + + "the fields " + answer.keySet() + "."); + assertTrue( + device.isNull("ismobile") == false + || device.has("ismobilenullreason"), + "The live cloud service answered " + IS_MOBILE + " as null " + + "with no reason beside it."); + for (String element : answer.keySet()) { + JSONObject values = answer.optJSONObject(element); + if (values == null) { + continue; + } + for (String property : values.keySet()) { + assertTrue( + wasAsked(asked, element, property), + "The live cloud service answered a license key alone " + + "with " + element + "." + property + ", which was " + + "not asked for."); + } + } + + // The name that is not a property is reported once, and the + // property answered as null is not reported. + verify(logger, times(1)).warn(argThat( + (String m) -> m.contains(NOT_A_PROPERTY) + && m.contains(IS_MOBILE) == false)); + } + + /** + * The fully qualified name of the first property of the first product + * the service lists, in name order, or null when it lists none. + */ + private static String firstProperty( + Map products) { + if (products == null) { + return null; + } + for (Map.Entry + product : new TreeMap<>(products).entrySet()) { + if (product.getValue().properties != null + && product.getValue().properties.isEmpty() == false) { + return product.getKey() + "." + + product.getValue().properties.get(0).name; + } + } + return null; + } + + /** + * True if the property of the element, or the property its null reason + * belongs to, is one of the names asked for, in any letter case. + */ + private static boolean wasAsked( + List asked, String element, String property) { + String suffix = "nullreason"; + for (String name : asked) { + if (name.equalsIgnoreCase(element + "." + property)) { + return true; + } + if (property.endsWith(suffix) && name.equalsIgnoreCase( + element + "." + property.substring( + 0, property.length() - suffix.length()))) { + return true; + } + } + return false; + } } diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index f7b274895..d0fa43b81 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -143,8 +143,11 @@ public void Process_LicenseKeyInCloudError_Masked() throws Exception { data.addEvidence("query.User-Agent", "iPhone"); data.process(); - assertFalse(data.getErrors().isEmpty()); - StringBuilder reported = new StringBuilder(); + // The response is a lazy property, so once processing has + // finished its error reaches only the reader. + RuntimeException read = assertThrows(RuntimeException.class, + () -> data.getFromElement(engine).getJsonResponse()); + StringBuilder reported = new StringBuilder(printed(read)); for (FlowError error : data.getErrors()) { reported.append(printed(error.getThrowable())); } diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineWarningsTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineWarningsTests.java new file mode 100644 index 000000000..b197784d3 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineWarningsTests.java @@ -0,0 +1,167 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.common.testhelpers.TestLogger; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import org.junit.jupiter.api.Test; + +import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Each entry in the warnings array of a response from the cloud service is + * logged as a warning, for the requests made when the engine is built and + * for those made when it processes. + */ +public class CloudRequestEngineWarningsTests + extends CloudRequestEngineTestsBase { + + private static final String LICENSE_KEY = "TESTKEYNOTREALABCDEFGHIJ1234"; + + private static final String MASKED = "************************1234"; + + public CloudRequestEngineWarningsTests() throws MalformedURLException { + super(); + } + + @Test + public void Process_EachWarningLoggedAsAWarning() throws Exception { + jsonResponse = "{\"device\":{\"value\":\"1\"}," + + "\"warnings\":[\"first warning\",\"second warning\"]}"; + configureMockedClient(); + + process(new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build()); + + assertLoggedOnce("first warning"); + assertLoggedOnce("second warning"); + } + + @Test + public void Build_WarningInAccessibleProperties_LoggedAsAWarning() + throws Exception { + accessiblePropertiesResponse = + "{\"Products\": {\"device\": {\"DataTier\": \"tier\"," + + "\"Properties\": [{\"Name\": \"value\",\"Type\": \"String\"," + + "\"Category\": \"Device\"}]}}," + + "\"warnings\":[\"start-up warning\"]}"; + configureMockedClient(); + + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build(); + + assertNotNull(engine.getPublicProperties()); + assertLoggedOnce("start-up warning"); + } + + /** + * The warnings are logged when the response also holds errors, which + * are thrown as before. + */ + @Test + public void Process_WarningBesideAnError_LoggedAndErrorRaised() + throws Exception { + jsonResponse = "{\"errors\":[\"an error\"]," + + "\"warnings\":[\"a warning\"]}"; + configureMockedClient(); + + String errors = process( + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build()); + + assertTrue(errors.contains("an error"), errors); + assertLoggedOnce("a warning"); + } + + /** + * A warning that names the license key shows it only by its last 4 + * characters, as an error does. + */ + @Test + public void Process_LicenseKeyInWarning_Masked() throws Exception { + jsonResponse = "{\"device\":{\"value\":\"1\"}," + + "\"warnings\":[\"License key " + LICENSE_KEY + + " expires soon\"]}"; + configureMockedClient(); + + process(new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .setLicenseKey(LICENSE_KEY) + .build()); + + for (String warning : warnings()) { + assertFalse(warning.contains(LICENSE_KEY), warning); + } + assertLoggedOnce("License key " + MASKED + " expires soon"); + } + + /** + * Process one request, and give the text of each error it raised. + */ + private String process(CloudRequestEngine engine) throws Exception { + StringBuilder errors = new StringBuilder(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(true) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + data.getErrors().forEach(e -> + errors.append(e.getThrowable().getMessage()).append('\n')); + // The response is a lazy property, so once processing has + // finished its error reaches only the reader. + try { + data.getFromElement(engine).getJsonResponse(); + } catch (RuntimeException e) { + errors.append(e.getMessage()).append('\n'); + } + } + return errors.toString(); + } + + private List warnings() { + List warnings = new ArrayList<>(); + for (TestLogger logger : loggerFactory.loggers) { + warnings.addAll(logger.warningsLogged); + } + return warnings; + } + + private void assertLoggedOnce(String expected) { + List warnings = warnings(); + long count = warnings.stream().filter(expected::equals).count(); + assertEquals(1, count, + "Expected the warning '" + expected + "' once in " + warnings); + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestFailedReasonTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestFailedReasonTests.java new file mode 100644 index 000000000..66d70a737 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestFailedReasonTests.java @@ -0,0 +1,271 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.common.testhelpers.TestLoggerFactory; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.factories.ElementDataFactory; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectDataBase; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.flowelements.AspectEngine; +import fiftyone.pipeline.engines.flowelements.AspectEngineBase; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * A Cloud Aspect Engine's property is missing with the CloudRequestFailed + * reason when the Cloud Request Engine's request for the Flow Data failed, + * naming the first engine of the Element Data, and when the engine's + * metadata is not loaded because the request for the property list failed. + */ +public class CloudRequestFailedReasonTests { + + private static final String CLOUD_REQUEST_FAILED = + "This is because an upstream cloud request failed, so no data was " + + "available to populate the property."; + + private final TestLoggerFactory loggerFactory = + new TestLoggerFactory(null); + + public static class TestData extends AspectDataBase { + public TestData( + Logger logger, + FlowData flowData, + AspectEngine engine) { + super( + logger, + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + } + } + + private class TestAspectEngine extends CloudAspectEngineBase { + TestAspectEngine() { + super( + loggerFactory.getLogger("TestAspectEngine"), + new ElementDataFactory() { + @Override + public TestData create( + FlowData flowData, + FlowElement flowElement) { + return new TestData( + loggerFactory.getLogger("TestData"), + flowData, + (AspectEngine) flowElement); + } + }); + } + + @Override + public String getElementDataKey() { + return "test"; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist(new ArrayList()); + } + + @Override + protected void unmanagedResourcesCleanup() { + } + + @Override + protected void processCloudEngine( + FlowData data, + TestData aspectData, + String json) { + } + } + + /** + * A Cloud Request Engine whose request fails, or answers with no part + * for the test engine, and which has the given accessible properties. + */ + private class TestRequestEngine + extends AspectEngineBase + implements CloudRequestEngine { + private final boolean fail; + private final Map + products; + + TestRequestEngine( + boolean fail, + Map products) { + super( + loggerFactory.getLogger("TestRequestEngine"), + new ElementDataFactory() { + @Override + public CloudRequestData create( + FlowData flowData, + FlowElement flowElement) { + return new CloudRequestData( + loggerFactory.getLogger("CloudRequestData"), + flowData, + (AspectEngine) flowElement); + } + }); + this.fail = fail; + this.products = products; + } + + @Override + public Map + getPublicProperties() { + return products; + } + + @Override + public String getDataSourceTier() { + return ""; + } + + @Override + public String getElementDataKey() { + return "cloud"; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist(new ArrayList()); + } + + @Override + public List getProperties() { + return Collections.emptyList(); + } + + @Override + protected void unmanagedResourcesCleanup() { + } + + @Override + protected void processEngine( + FlowData data, + CloudRequestData aspectData) throws IOException { + aspectData.setProcessStarted(true); + if (fail) { + throw new IOException("The cloud service could not be reached."); + } + aspectData.setJsonResponse("{ \"other\": {} }"); + } + } + + @Test + public void RequestFailed_CloudRequestFailed() throws Exception { + PropertyMissingException e = readMissing(true, products()); + + assertEquals("CloudRequestFailed", e.getReason().name(), + e.getMessage()); + assertEquals( + "Property 'ismobile' not found in data for element 'test'. " + + CLOUD_REQUEST_FAILED, + e.getMessage()); + } + + @Test + public void RequestSucceeded_NotCloudRequestFailed() throws Exception { + PropertyMissingException e = readMissing(false, products()); + + assertEquals( + "PropertyNotAccessibleWithResourceKey", + e.getReason().name(), + e.getMessage()); + } + + @Test + public void PropertyListRequestFailed_CloudRequestFailed() + throws Exception { + PropertyMissingException e = readMissing(false, null); + + assertEquals("CloudRequestFailed", e.getReason().name(), + e.getMessage()); + assertEquals( + "Property 'ismobile' not found in data for element 'test'. " + + CLOUD_REQUEST_FAILED, + e.getMessage()); + } + + /** + * Process one Flow Data, with process exceptions suppressed, and read + * the property the response does not hold. + */ + private PropertyMissingException readMissing( + boolean fail, + Map products) + throws Exception { + TestAspectEngine engine = new TestAspectEngine(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(true) + .addFlowElement(new TestRequestEngine(fail, products)) + .addFlowElement(engine) + .build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.user-agent", "iPhone"); + flowData.process(); + TestData data = flowData.getFromElement(engine); + return assertThrows( + PropertyMissingException.class, + () -> data.get("ismobile")); + } + } + + /** + * Products holding one test property, so that the engine's metadata is + * loaded and does not list the property read. + */ + private static Map + products() { + List properties = + new ArrayList<>(); + properties.add(new AccessiblePropertyMetaData.PropertyMetaData( + "hardwarevendor", "String", null, null)); + AccessiblePropertyMetaData.ProductMetaData product = + new AccessiblePropertyMetaData.ProductMetaData(); + product.properties = properties; + Map products = + new HashMap<>(); + products.put("test", product); + return products; + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestLazyTaskTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestLazyTaskTests.java new file mode 100644 index 000000000..dcb93d2d7 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestLazyTaskTests.java @@ -0,0 +1,489 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.pipeline.cloudrequestengine.CloudRequestException; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.data.factories.ElementDataFactory; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectDataBase; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.flowelements.AspectEngine; +import fiftyone.pipeline.engines.services.MissingPropertyReason; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.jupiter.api.Test; +import org.mockito.AdditionalAnswers; +import org.mockito.ArgumentMatchers; +import org.slf4j.Logger; + +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; + +/** + * The JSON response is a lazy property. The engine builds the request while + * it processes, starts it as a task, and returns without waiting. Readers + * wait on the task, and a failure met during processing is recorded once as + * the engine's own error. + */ +public class CloudRequestLazyTaskTests extends CloudRequestEngineTestsBase { + + public CloudRequestLazyTaskTests() throws MalformedURLException { + super(); + } + + public static class TestData extends AspectDataBase { + public TestData( + Logger logger, + FlowData flowData, + AspectEngine engine) { + super( + logger, + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + } + } + + private class TestAspectEngine extends CloudAspectEngineBase { + private final String key; + + TestAspectEngine(String key) { + super( + loggerFactory.getLogger("TestAspectEngine"), + new ElementDataFactory() { + @Override + public TestData create( + FlowData flowData, + FlowElement flowElement) { + return new TestData( + loggerFactory.getLogger("TestData"), + flowData, + (AspectEngine) flowElement); + } + }); + this.key = key; + } + + @Override + public String getElementDataKey() { + return key; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist(new ArrayList()); + } + + @Override + protected void unmanagedResourcesCleanup() { + } + + @Override + protected void processCloudEngine( + FlowData data, + TestData aspectData, + String json) { + } + } + + private static final long WAIT_SECONDS = 10; + + /** + * How long a call that should wait is given to return anyway. + */ + private static final long HELD_MILLISECONDS = 300; + + private static final int READERS = 8; + + private static boolean await(CountDownLatch latch) { + try { + return latch.await(WAIT_SECONDS, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + /** + * Meets the failed request first, then holds its add of the failure to + * the errors until the second reader has finished processing, as two + * members of a parallel group can. + */ + private class FirstReader extends TestAspectEngine { + private final CountDownLatch adding; + private final CountDownLatch release; + + FirstReader(CountDownLatch adding, CountDownLatch release) { + super("first"); + this.adding = adding; + this.release = release; + } + + @Override + protected void processEngine(FlowData data, TestData aspectData) { + FlowData holding = mock( + FlowData.class, + AdditionalAnswers.delegatesTo(data)); + doAnswer(invocation -> { + adding.countDown(); + await(release); + data.addError( + invocation.getArgument(0), + invocation.>getArgument(1)); + return null; + }).when(holding).addError(any(Throwable.class), any()); + super.processEngine(holding, aspectData); + } + } + + /** + * Reads the response once the first reader is adding the failure. + */ + private class SecondReader extends TestAspectEngine { + private final CountDownLatch adding; + private final CountDownLatch release; + + SecondReader(CountDownLatch adding, CountDownLatch release) { + super("second"); + this.adding = adding; + this.release = release; + } + + @Override + protected void processEngine(FlowData data, TestData aspectData) { + try { + await(adding); + super.processEngine(data, aspectData); + } finally { + release.countDown(); + } + } + } + + /** + * Hold the data request until processing has returned, and record + * whether it did. The wait is bounded so an engine that waits for the + * response fails the test rather than hanging. + */ + private void holdDataRequest( + CountDownLatch processReturned, + AtomicBoolean answeredAfterProcess) throws Exception { + doAnswer(invocation -> { + URL url = ((HttpURLConnection) invocation.getArgument(0)).getURL(); + if (url.getPath().endsWith("properties")) { + return accessiblePropertiesResponse; + } + if (url.getPath().endsWith("evidencekeys")) { + return evidenceKeysResponse; + } + answeredAfterProcess.set( + processReturned.await(10, TimeUnit.SECONDS)); + return jsonResponse; + }) + .when(httpClient) + .postData( + any(HttpURLConnection.class), + ArgumentMatchers.anyMap(), + (byte[]) any()); + } + + @Test + public void Process_ReturnsBeforeTheResponse() throws Exception { + configureMockedClient(); + CountDownLatch processReturned = new CountDownLatch(1); + AtomicBoolean answeredAfterProcess = new AtomicBoolean(false); + holdDataRequest(processReturned, answeredAfterProcess); + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + + data.process(); + processReturned.countDown(); + + assertEquals(jsonResponse, + data.getFromElement(engine).getJsonResponse()); + assertTrue(answeredAfterProcess.get(), + "Processing waited for the cloud response."); + } finally { + processReturned.countDown(); + } + } + + @Test + public void FailedRequest_AfterProcessing_ReachesOnlyTheReader() + throws Exception { + jsonResponse = "{\"errors\":[\"The request failed.\"]}"; + configureMockedClient(); + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(true) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + + assertThrows(CloudRequestException.class, + () -> data.getFromElement(engine).getJsonResponse()); + assertThrows(CloudRequestException.class, + () -> data.getFromElement(engine).getJsonResponse(), + "Every reader gets the failure."); + assertTrue(data.getErrors() == null || data.getErrors().isEmpty(), + "A failure met after processing is not recorded."); + } + } + + @Test + public void FailedRequest_MetWhileProcessing_RecordedOnceAsEngineError() + throws Exception { + jsonResponse = "{\"errors\":[\"The request failed.\"]}"; + configureMockedClient(); + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(true) + .addFlowElement(engine) + .addFlowElement(new TestAspectEngine("first")) + .addFlowElement(new TestAspectEngine("second")) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + + List errors = new ArrayList<>(data.getErrors()); + assertEquals(1, errors.size(), errors.toString()); + assertSame(engine, errors.get(0).getFlowElement()); + assertInstanceOf(CloudRequestException.class, + errors.get(0).getThrowable()); + } + } + + @Test + @SuppressWarnings("rawtypes") + public void FailedRequest_EveryParallelReaderMarkedFailed() + throws Exception { + jsonResponse = "{\"errors\":[\"The request failed.\"]}"; + configureMockedClient(); + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build(); + CountDownLatch adding = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + FirstReader first = new FirstReader(adding, release); + SecondReader second = new SecondReader(adding, release); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(true) + .addFlowElement(engine) + .addFlowElementsParallel(new FlowElement[]{first, second}) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + + assertEquals(1, data.getErrors().size(), + data.getErrors().toString()); + for (TestAspectEngine reader : Arrays.asList(first, second)) { + PropertyMissingException e = assertThrows( + PropertyMissingException.class, + () -> data.getFromElement(reader).get("value")); + assertEquals( + MissingPropertyReason.CloudRequestFailed, + e.getReason(), + reader.getElementDataKey() + " was not marked as " + + "failed before the failure was recorded"); + } + } finally { + release.countDown(); + } + } + + @Test + public void ConcurrentReaders_OneRequestAndTheSameResponse() + throws Exception { + configureMockedClient(); + CountDownLatch release = new CountDownLatch(1); + AtomicInteger dataRequests = new AtomicInteger(); + doAnswer(invocation -> { + URL url = ((HttpURLConnection) invocation.getArgument(0)).getURL(); + if (url.getPath().endsWith("properties")) { + return accessiblePropertiesResponse; + } + if (url.getPath().endsWith("evidencekeys")) { + return evidenceKeysResponse; + } + dataRequests.incrementAndGet(); + await(release); + return jsonResponse; + }) + .when(httpClient) + .postData( + any(HttpURLConnection.class), + ArgumentMatchers.anyMap(), + (byte[]) any()); + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build(); + ExecutorService readers = Executors.newFixedThreadPool(READERS); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + CloudRequestData requestData = data.getFromElement(engine); + + List> reads = new ArrayList<>(); + for (int i = 0; i < READERS; i++) { + reads.add(readers.submit(requestData::getJsonResponse)); + } + Thread.sleep(HELD_MILLISECONDS); + for (Future read : reads) { + assertFalse(read.isDone(), + "A reader did not wait for the response"); + } + release.countDown(); + for (Future read : reads) { + assertEquals(jsonResponse, + read.get(WAIT_SECONDS, TimeUnit.SECONDS)); + } + assertEquals(1, dataRequests.get(), + "The request was made more than once"); + } finally { + release.countDown(); + readers.shutdownNow(); + } + } + + @Test + public void Task_DoesNotReadTheFlowData() throws Exception { + configureMockedClient(); + CountDownLatch processReturned = new CountDownLatch(1); + AtomicBoolean answeredAfterProcess = new AtomicBoolean(false); + holdDataRequest(processReturned, answeredAfterProcess); + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + FlowData watched = mock( + FlowData.class, + AdditionalAnswers.delegatesTo(data)); + engine.process(watched); + CloudRequestData requestData = data.getOrAdd( + engine.getTypedDataKey(), + flowData -> null); + clearInvocations(watched); + + processReturned.countDown(); + assertEquals(jsonResponse, requestData.getJsonResponse()); + assertTrue(answeredAfterProcess.get()); + verifyNoInteractions(watched); + } finally { + processReturned.countDown(); + } + } + + @Test + public void Close_WaitsForARequestStillRunning() throws Exception { + configureMockedClient(); + CountDownLatch processReturned = new CountDownLatch(1); + AtomicBoolean answeredAfterProcess = new AtomicBoolean(false); + holdDataRequest(processReturned, answeredAfterProcess); + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build(); + ExecutorService closer = Executors.newSingleThreadExecutor(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + CloudRequestData requestData = data.getFromElement(engine); + + Future closed = closer.submit(() -> { + engine.close(); + return null; + }); + try { + assertThrows(TimeoutException.class, + () -> closed.get( + HELD_MILLISECONDS, + TimeUnit.MILLISECONDS), + "Closing the engine did not wait for its request"); + } finally { + processReturned.countDown(); + } + closed.get(WAIT_SECONDS, TimeUnit.SECONDS); + assertEquals(jsonResponse, requestData.getJsonResponse()); + } finally { + processReturned.countDown(); + closer.shutdownNow(); + } + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java index 6bb87c5fe..88f18402f 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java @@ -29,32 +29,27 @@ *

    * Every name is tried as an environment variable first and then as a * system property, so a developer can export one and a build can pass one - * on the command line with {@code -D}. There is no fallback to a skip, - * because a cloud test that quietly passes with no key reports success - * having called nothing. + * on the command line with {@code -D}. Each key is read under its one + * _51DEGREES_ name and no other. A test with no key is skipped, naming + * the variable to set, rather than passed having called nothing. */ final class LiveCloudKeys { /** - * The resource key names, in the order they are tried. + * The name the resource key is read from. */ static final String[] RESOURCE_KEY_NAMES = { "_51DEGREES_RESOURCE_KEY", - "_51DEGREES_RESOURCE_KEY_BESPOKE", - "TestResourceKey", - "SUPER_RESOURCE_KEY", }; /** - * The licence key names, in the order they are tried. A licence key - * is optional, because an account can hold its products on the - * resource key alone. + * The name the license key is read from. A license key is optional, + * because an account can hold its products on the resource key alone. + * It is the license for the cloud products, and not + * _51DEGREES_LICENSE_KEY, which holds the license for the data files. */ static final String[] LICENCE_KEY_NAMES = { - "_51DEGREES_LICENSE_KEY", "_51DEGREES_LICENSE_KEY_BESPOKE", - "_51DEGREES_LICENSE_KEY_CLOUDV5BESPOKE", - "TestLicenseKey", }; private LiveCloudKeys() { @@ -78,16 +73,16 @@ static String find(String[] names) { } /** - * What to fail with when no resource key is set, naming every place - * that was looked and what the key has to be entitled to. + * Why the test is skipped when no resource key is set, naming the + * variable to set and what the key has to be entitled to. */ static String noResourceKey() { return "This test calls the live 51Degrees cloud service, so it " - + "needs a resource key and there is nothing to fall back on. " - + "None of " + String.join(", ", RESOURCE_KEY_NAMES) + " is set, " - + "as an environment variable or as a system property, tried in " - + "that order. The key has to carry the device product, because " + + "needs a resource key, and it is skipped because " + + String.join(", ", RESOURCE_KEY_NAMES) + " is not set as an " + + "environment variable or as a system property. " + + "The key has to carry the device product, because " + "the test asks the service for device detection properties. " - + "Create a key at https://configure.51degrees.com."; + + "Create a key at https://51degrees.com/configure."; } } diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeysTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeysTests.java new file mode 100644 index 000000000..103fa66fb --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeysTests.java @@ -0,0 +1,59 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Each key the live cloud tests use is read under its one _51DEGREES_ name, + * and no other name is read in its place. + */ +public class LiveCloudKeysTests { + + @Test + public void resourceKey_ReadUnderItsOwnNameOnly() { + assertArrayEquals(new String[]{"_51DEGREES_RESOURCE_KEY"}, + LiveCloudKeys.RESOURCE_KEY_NAMES); + } + + /** + * The test on a license key alone needs a license for a cloud product. + * _51DEGREES_LICENSE_KEY holds the data file license, so it is not + * read here. + */ + @Test + public void licenceKey_IsTheCloudLicence() { + assertArrayEquals(new String[]{"_51DEGREES_LICENSE_KEY_BESPOKE"}, + LiveCloudKeys.LICENCE_KEY_NAMES); + } + + @Test + public void noResourceKey_NamesTheVariable() { + assertTrue(LiveCloudKeys.noResourceKey().contains( + "_51DEGREES_RESOURCE_KEY is not set"), + LiveCloudKeys.noResourceKey()); + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/WritesAfterProcessingTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/WritesAfterProcessingTests.java new file mode 100644 index 000000000..fe5b881a6 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/WritesAfterProcessingTests.java @@ -0,0 +1,114 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.data.AspectDataBase; +import org.junit.jupiter.api.Test; + +import java.net.MalformedURLException; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * The check, for the Flow Element in this module, the + * {@link CloudRequestEngineDefault}, that processes it, then writes to its + * Element Data and expects the error. The cloud service is a stand-in that + * gives a fixed response. + * @see Specification + */ +public class WritesAfterProcessingTests extends CloudRequestEngineTestsBase { + + public WritesAfterProcessingTests() throws MalformedURLException { + super(); + } + + static void assertWritesFail(ElementData data) { + assertThrows( + IllegalStateException.class, + () -> data.put("after", "value"), + "put"); + assertThrows( + IllegalStateException.class, + () -> data.populateFromMap( + Collections.singletonMap("after", "value")), + "populateFromMap"); + } + + /** + * The write methods every Aspect Data has fail as well. + */ + static void assertAspectWritesFail(AspectDataBase data) { + assertThrows( + IllegalStateException.class, + data::setCloudRequestFailed, + "setCloudRequestFailed"); + assertThrows( + IllegalStateException.class, + () -> data.addEngine(null), + "addEngine"); + assertThrows( + IllegalStateException.class, + () -> data.addProcessCallable(null), + "addProcessCallable"); + } + + @Test + public void cloudRequestEngine_ElementDataRefusesWrites() + throws Exception { + configureMockedClient(); + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + assertEquals(0, data.getErrors().size(), + String.valueOf(data.getErrors())); + + CloudRequestData own = data.getFromElement(engine); + assertEquals(jsonResponse, own.getJsonResponse()); + assertWritesFail(own); + assertAspectWritesFail(own); + assertThrows( + IllegalStateException.class, + () -> own.setJsonResponse("{}"), + "setJsonResponse"); + assertThrows( + IllegalStateException.class, + () -> own.setProcessStarted(true), + "setProcessStarted"); + } + } +} diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/Constants.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/Constants.java index 205fc3f4e..15bc17786 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/Constants.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/Constants.java @@ -48,5 +48,17 @@ public class Constants { public static final String EVIDENCE_PROTOCOL = EVIDENCE_HTTPHEADER_PREFIX + EVIDENCE_SEPERATOR + "protocol"; - public static final String EVIDENCE_SESSION_KEY = EVIDENCE_SESSION_PREFIX + EVIDENCE_SEPERATOR + "session"; + public static final String EVIDENCE_SESSION_KEY = EVIDENCE_SESSION_PREFIX + EVIDENCE_SEPERATOR + "session"; + + /** + * Whether a Pipeline closes its elements when it is closed, unless its + * builder is told otherwise. + */ + public static final boolean DEFAULT_AUTO_CLOSE_ELEMENTS = false; + + /** + * Whether a Pipeline records the exceptions thrown while processing as + * errors rather than throwing them, unless its builder is told otherwise. + */ + public static final boolean DEFAULT_SUPPRESS_PROCESS_EXCEPTIONS = false; } diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/DataBase.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/DataBase.java index dd765ecb5..af87e7ac0 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/DataBase.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/DataBase.java @@ -80,6 +80,14 @@ public void put(String key, Object value) { data.put(key, value); } + /** + * Remove every value, so a pooled instance holds nothing of the request + * it served before. + */ + protected void clearValues() { + data.clear(); + } + @Override public Map asKeyMap() { return Collections.unmodifiableMap(data); diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementDataBase.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementDataBase.java index 937b3c590..e17a6f55a 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementDataBase.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementDataBase.java @@ -40,11 +40,37 @@ *

    * @see Specification * @see Thread safety + *

    + * Element Data is written only by the Flow Element that provides it, or the + * Flow Elements that share it, and only while they process. Once a Flow + * Element that writes it has finished, {@link #put(String, Object)}, + * {@link #populateFromMap(Map)} and every setter built on them fail with an + * {@link IllegalStateException}. Values kept once they have been read, with + * {@link #keepValues(Map)}, are not writes. Like the Flow Data that holds + * it, Element Data is not safe to use from several threads at once. + *

    + * Element Data that can be pooled goes back to its Flow Element's pool when + * the Flow Data that holds it is closed, and may then be given to another + * request. Keeping a reference to Element Data after its Flow Data is closed + * is not supported, and reads do not check for it. + * @see Specification + * @see Reusing Element Data */ public abstract class ElementDataBase extends DataBase implements ElementData { private Pipeline pipeline; + /** + * Set once a Flow Element that writes this instance has finished, after + * which a write fails. A write tests it, and nothing waits on it. + */ + private volatile boolean finished = false; + + private static final String FINISHED_MESSAGE = + "This Element Data's Flow Element has finished processing, so it " + + "is read only. Element Data is written only by the Flow Element " + + "that provides it, while it processes."; + /** * Constructs a new instance with a non-thread-safe, case-insensitive * {@link Map} as the underlying storage. @@ -82,4 +108,164 @@ public Pipeline getPipeline() { public void setPipeline(Pipeline pipeline) { this.pipeline = pipeline; } + + /** + * Record that a Flow Element that writes this instance has finished, so + * that a later write fails. Called by the Flow Element when it finishes + * processing, whether or not it succeeded. + */ + public void finishWriting() { + finished = true; + } + + /** + * Open this instance for writing again, as a Flow Element that shares it + * does when it starts to process. + */ + public void startWriting() { + finished = false; + } + + /** + * @return true once a Flow Element that writes this instance has + * finished, until one that shares it starts to process or its pool gives + * it to another Flow Data + */ + public boolean isFinishedWriting() { + return finished; + } + + /** + * Where this instance goes when the Flow Data that holds it is closed, or + * null when it is not pooled. Set by the Flow Element that gave it out. + */ + private volatile Runnable onRelease = null; + + /** + * Whether this instance can be given out again once the Flow Data that + * held it is closed. An implementation whose only state is its values + * returns true. One that keeps other state of the request, or puts + * values in its constructor, also puts that back in {@link #onReuse()}. + * One that a caller may keep after the Flow Data is closed keeps the + * default. + * @return false unless an implementation opts in + */ + protected boolean isPoolable() { + return false; + } + + /** + * @return true if this instance can be returned to its Flow Element's + * pool + */ + public boolean canBePooled() { + return isPoolable(); + } + + /** + * Set what happens to this instance when the Flow Data that holds it is + * closed. Used by the Flow Element that gave it out. + * @param onRelease run once on release, or null for none + */ + public void setOnRelease(Runnable onRelease) { + this.onRelease = onRelease; + } + + /** + * Keep this instance out of its pool, as a cache that holds it does. + */ + public void keepFromPool() { + onRelease = null; + } + + /** + * Whether the task of a lazy property may still write this instance. An + * implementation with a lazy property whose value a task works out + * returns true until the task has finished. + * @return false unless an implementation has a lazy task running + * @see Lazy properties + */ + protected boolean hasLazyTaskRunning() { + return false; + } + + /** + * Called when the Flow Data that holds this instance is closed. A pooled + * instance goes back to its Flow Element's pool, once only. One whose + * lazy property's task is still running is left for the garbage + * collector instead, because the task may still write it. + */ + public void release() { + Runnable release = onRelease; + onRelease = null; + if (release != null && hasLazyTaskRunning() == false) { + release.run(); + } + } + + /** + * Make a pooled instance ready for the next Flow Data, with no values + * and open for writing. + * @param flowData the Flow Data it is given to + */ + public void reuse(FlowData flowData) { + clearValues(); + finished = false; + pipeline = flowData.getPipeline(); + onReuse(); + } + + /** + * Put back, as the constructor left it, any state other than the values + * that this instance kept from the request it served before. Called when + * a pooled instance is given to the next Flow Data. + */ + protected void onReuse() { + } + + /** + * Store a value, which only the Flow Element that provides this instance, + * or one that shares it, does while it processes. + * @throws IllegalStateException if the Flow Element has finished + */ + @Override + public void put(String key, Object value) { + if (finished) { + throw new IllegalStateException(FINISHED_MESSAGE); + } + super.put(key, value); + } + + /** + * Store values, which only the Flow Element that provides this instance, + * or one that shares it, does while it processes. + * @throws IllegalStateException if the Flow Element has finished + */ + @Override + public void populateFromMap(Map values) { + checkWriting(); + super.populateFromMap(values); + } + + /** + * Keep values this instance has read, such as those read from a results + * instance, so they are not read again. Keeping them is not a write, so + * it works once the Flow Element has finished. + * @param values the values read + */ + protected void keepValues(Map values) { + super.populateFromMap(values); + } + + /** + * Fail a write once a Flow Element that writes this instance has + * finished. A write method that does not go through + * {@link #put(String, Object)} calls it first. + * @throws IllegalStateException if the Flow Element has finished + */ + protected void checkWriting() { + if (finished) { + throw new IllegalStateException(FINISHED_MESSAGE); + } + } } diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/EvidenceKeyFilterWhitelist.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/EvidenceKeyFilterWhitelist.java index 53cd41464..0885ef1c9 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/EvidenceKeyFilterWhitelist.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/EvidenceKeyFilterWhitelist.java @@ -36,13 +36,12 @@ public class EvidenceKeyFilterWhitelist implements EvidenceKeyFilter { /** * Construct a new instance using the list of evidence keys provided. + * Keys are compared ignoring case, as Evidence keys are. * By default, all keys will have the same order of precedence. * @param whitelist evidence keys to add to the whitelist */ public EvidenceKeyFilterWhitelist(List whitelist) { - this.whitelist = new TreeMap<>(); - addValues(whitelist); - this.comparator = null; + this(whitelist, String.CASE_INSENSITIVE_ORDER); } /** @@ -66,12 +65,11 @@ public EvidenceKeyFilterWhitelist( * precedence provided. * The order of precedence of each key is given by the value of the * key/value pair. + * Keys are compared ignoring case, as Evidence keys are. * @param whitelist evidence keys to add to the whitelist */ public EvidenceKeyFilterWhitelist(Map whitelist) { - this.whitelist = new TreeMap<>(); - addValues(whitelist); - this.comparator = null; + this(whitelist, String.CASE_INSENSITIVE_ORDER); } /** diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/FlowData.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/FlowData.java index 763da2bb3..20f624592 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/FlowData.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/FlowData.java @@ -22,9 +22,11 @@ package fiftyone.pipeline.core.data; +import fiftyone.pipeline.core.exceptions.PipelineDataException; import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.exceptions.AggregateException; import java.util.Collection; import java.util.List; @@ -33,15 +35,31 @@ /** * A closeable structure to contain the inputs and outputs of {@link Pipeline} * processing. + *

    + * A Flow Data is used by one thread at a time. It is not safe to use from + * several threads at once, even once processing has finished, and a caller + * that shares one across threads is responsible for any locking. + *

    + * Closing a Flow Data may return its Element Data to an object pool, from + * which another request can take it. Keeping a reference to Element Data + * after its Flow Data is closed is not supported. * @see Specification + * @see Thread safety */ public interface FlowData extends AutoCloseable { /** * Process this FlowData via the owning Pipeline. Once this method returns * the FlowData will not be changed further and cannot be processed again. + * It is read only from then on, apart from lazy properties, so adding + * Evidence, Element Data or errors throws an + * {@link IllegalStateException}. + * @throws IllegalStateException if this FlowData has already been + * processed, or its Pipeline has been closed + * @throws AggregateException if a Flow Element failed and the Pipeline + * does not suppress process exceptions, holding each failure */ - void process(); + void process() throws IllegalStateException, AggregateException; /** * If this flag is set to true, future FlowElements will not process this @@ -128,26 +146,37 @@ public interface FlowData extends AutoCloseable { * Get the {@link ElementData} for the requested aspect. * @param key the aspect to get * @param type type of the aspect - * @return aspect's {@link ElementData} or null if not present + * @return aspect's {@link ElementData} + * @throws IllegalStateException if this FlowData has not been processed + * @throws IllegalArgumentException if the key is null + * @throws java.util.NoSuchElementException if no element added data + * under the key */ - T get(TypedKey key); + T get(TypedKey key) + throws IllegalStateException, IllegalArgumentException, + java.util.NoSuchElementException; /** * Get the {@link ElementData} for the requested data key. * @param key to the aspect to get - * @return aspect's {@link ElementData} or null if not present + * @return aspect's {@link ElementData} + * @throws IllegalStateException if this FlowData has not been processed + * @throws java.util.NoSuchElementException if no element added data + * under the key */ - ElementData get(String key); + ElementData get(String key) + throws IllegalStateException, java.util.NoSuchElementException; /** * Get the {@link ElementData} for the requested {@link ElementData} - * implementation. If multiple instances of the type exist then an exception - * is thrown. + * implementation. * @param type the type of {@link ElementData} to return * @param the type of {@link ElementData} to return - * @return aspect's {@link ElementData} or null if the type is not present + * @return aspect's {@link ElementData} + * @throws PipelineDataException if no data of the type is present, or + * more than one instance is */ - T get(Class type); + T get(Class type) throws PipelineDataException; /** * Get the ElementData for the requested {@link FlowElement}. This method @@ -157,8 +186,11 @@ public interface FlowData extends AutoCloseable { * @param element {@link FlowElement} to get the data for * @param {@link ElementData} stored by the element * @return element data stored by the element + * @throws IllegalStateException if this FlowData has not been processed + * @throws IllegalArgumentException if the element is null */ - T getFromElement(FlowElement element); + T getFromElement(FlowElement element) + throws IllegalStateException, IllegalArgumentException; /** * Get the specified property as the specified type. @@ -166,8 +198,12 @@ public interface FlowData extends AutoCloseable { * @param key the name of the property to get * @param type the type to return the property value as * @return the property value + * @throws PipelineDataException if this FlowData has not been processed, + * or no element of the Pipeline has a property of that name + * @throws ClassCastException if the value is not of the type given */ - T getAs(String key, Class type); + T getAs(String key, Class type) + throws PipelineDataException, ClassCastException; /** * Check if the flow data contains an item with the specified key name and @@ -183,43 +219,67 @@ public interface FlowData extends AutoCloseable { * Get the specified property as a string. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed, + * or no element of the Pipeline has a property of that name + * @throws ClassCastException if the value is not a string */ - String getAsString(String propertyName); + String getAsString(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get the specified property as a boolean. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed, + * or no element of the Pipeline has a property of that name + * @throws ClassCastException if the value is not a boolean */ - Boolean getAsBool(String propertyName); + Boolean getAsBool(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get the specified property as an integer. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed, + * or no element of the Pipeline has a property of that name + * @throws ClassCastException if the value is not an integer */ - Integer getAsInt(String propertyName); + Integer getAsInt(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get the specified property as a long. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed, + * or no element of the Pipeline has a property of that name + * @throws ClassCastException if the value is not a long */ - Long getAsLong(String propertyName); + Long getAsLong(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get the specified property as a float. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed, + * or no element of the Pipeline has a property of that name + * @throws ClassCastException if the value is not a float */ - Float getAsFloat(String propertyName); + Float getAsFloat(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get the specified property as a double. * @param propertyName the name of the property to get * @return the property value + * @throws PipelineDataException if this FlowData has not been processed, + * or no element of the Pipeline has a property of that name + * @throws ClassCastException if the value is not a double */ - Double getAsDouble(String propertyName); + Double getAsDouble(String propertyName) + throws PipelineDataException, ClassCastException; /** * Get or add the specified element data to the internal map. @@ -260,12 +320,16 @@ T getOrAdd( Iterable elementDataAsIterable(); /** - * Get all element data values that match the specified predicate. + * Get all element data values that match the specified predicate. The + * properties of an element that added no data, and properties with no + * value, are left out. * @param matcher if a property passed to this matcher returns true then it * will be included in the results * @return all the element data values that match the matcher + * @throws IllegalStateException if this FlowData has not been processed */ - Map getWhere(PropertyMatcher matcher); + Map getWhere(PropertyMatcher matcher) + throws IllegalStateException; /** * Generate a {@link DataKey} that contains the evidence data from this diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/EvidenceDefault.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/EvidenceDefault.java index 16a3a71a4..4979146e4 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/EvidenceDefault.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/EvidenceDefault.java @@ -26,14 +26,23 @@ import fiftyone.pipeline.core.data.Evidence; import org.slf4j.Logger; +import java.util.Map; import java.util.concurrent.ConcurrentSkipListMap; /** - * Default implementation of the {@link Evidence} interface. + * Default implementation of the {@link Evidence} interface. Once the Flow + * Data that holds it has finished processing, a write fails with an + * {@link IllegalStateException}. * @see Specification */ class EvidenceDefault extends DataBase implements Evidence { + /** + * Set when the Flow Data that holds this Evidence finishes processing. A + * write tests it, and nothing waits on it. + */ + private volatile boolean finished = false; + /** * Construct a new instance of the internal {@link Evidence} class using a * case insensitive {@link ConcurrentSkipListMap} as the backing map. @@ -41,4 +50,30 @@ class EvidenceDefault extends DataBase implements Evidence { EvidenceDefault(Logger logger) { super(logger, new ConcurrentSkipListMap<>(String.CASE_INSENSITIVE_ORDER)); } + + /** + * Make this Evidence read only, because its Flow Data has finished + * processing. + */ + void finish() { + finished = true; + } + + @Override + public void put(String key, Object value) { + checkNotFinished(); + super.put(key, value); + } + + @Override + public void populateFromMap(Map values) { + checkNotFinished(); + super.populateFromMap(values); + } + + private void checkNotFinished() { + if (finished) { + throw new IllegalStateException(FlowDataDefault.FINISHED_MESSAGE); + } + } } diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowDataDefault.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowDataDefault.java index f8758e126..1f7655824 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowDataDefault.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowDataDefault.java @@ -33,6 +33,7 @@ import java.util.*; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static fiftyone.pipeline.util.Check.getNotNull; @@ -46,12 +47,22 @@ class FlowDataDefault implements FlowData { private final Logger logger; - private final Object lock = new Object(); private final PipelineInternal pipeline; + /** + * The errors added. While a parallel group runs, its members add their + * errors under the lock on this collection, so they appear in the order + * the members add them. At every other time an add takes no lock. + */ private final Collection errors; + /** + * The parallel groups running on this Flow Data, so that errors are + * added under the lock only while one runs. + */ + private final AtomicInteger groupsRunning = new AtomicInteger(0); + private final TypedKeyMap data; private final Evidence evidence; @@ -60,6 +71,22 @@ class FlowDataDefault implements FlowData { private final AtomicBoolean processed = new AtomicBoolean(false); + /** + * Set by the first call to close, so later calls do nothing. + */ + private final AtomicBoolean closed = new AtomicBoolean(false); + + /** + * Set when processing ends, after which the Flow Data is read only. A + * write tests it, and nothing waits on it. + */ + private volatile boolean finished = false; + + static final String FINISHED_MESSAGE = + "This Flow Data has finished processing, so it is read only. " + + "Add Evidence before processing, and Element Data and errors only " + + "while a Flow Element processes."; + FlowDataDefault(Logger logger, Pipeline pipeline, Evidence evidence) { this.logger = logger; this.pipeline = (PipelineInternal) pipeline; @@ -86,7 +113,26 @@ public void process() { } else if (pipeline.isClosed()) { throw new IllegalStateException("Pipeline has been closed."); } else { - pipeline.process(this); + try { + pipeline.process(this); + } finally { + finished = true; + // Evidence made by this package refuses writes from now on + // too, including through getEvidence(). + if (evidence instanceof EvidenceDefault) { + ((EvidenceDefault) evidence).finish(); + } + } + } + } + + /** + * Fail a write once processing has finished. + * @throws IllegalStateException if processing has finished + */ + private void checkNotFinished() { + if (finished) { + throw new IllegalStateException(FINISHED_MESSAGE); } } @@ -100,6 +146,20 @@ public void stop() { stop = true; } + /** + * Called by a parallel group before its members start. + */ + void groupStarted() { + groupsRunning.incrementAndGet(); + } + + /** + * Called by a parallel group once every member has stopped. + */ + void groupStopped() { + groupsRunning.decrementAndGet(); + } + @Override public Pipeline getPipeline() { return pipeline; @@ -112,7 +172,14 @@ public Collection getErrors() { @Override public void addError(FlowError error) { - errors.add(error); + checkNotFinished(); + if (groupsRunning.get() > 0) { + synchronized (errors) { + errors.add(error); + } + } else { + errors.add(error); + } if (logger.isErrorEnabled()) { String message = "Error occurred during processing"; @@ -121,7 +188,7 @@ public void addError(FlowError error) { error.getFlowElement().getClass().getSimpleName() + "-" + error.getFlowElement().hashCode(); } - logger.error(message); + logger.error(message, error.getThrowable()); } } @@ -163,6 +230,7 @@ public List getDataKeys() { @Override public FlowData addEvidence(String key, Object value) { + checkNotFinished(); evidence.put(key, value); if (logger.isDebugEnabled()) { logger.debug("FlowData '" + hashCode() + "' set evidence " + @@ -173,6 +241,7 @@ public FlowData addEvidence(String key, Object value) { @Override public FlowData addEvidence(Map values) { + checkNotFinished(); for (Map.Entry entry : values.entrySet()) { addEvidence(entry.getKey(), entry.getValue()); } @@ -191,7 +260,7 @@ public T get(TypedKey key) { @Override public ElementData get(String key) { if (processed.get() == false) { - throw new RuntimeException("This instance has not yet been processed"); + throw new IllegalStateException("This instance has not yet been processed"); } return data.get(new TypedKeyDefault(key, ElementData.class)); @@ -281,16 +350,12 @@ public T getOrAdd( TypedKey typedKey = new TypedKeyDefault<>( elementDataKey, ElementData.class); + // No lock. Members of a parallel group each add their own Element + // Data with one compare and swap, and making an instance never + // waits for another member. if (data.containsKey(elementDataKey) == false) { - if (pipeline.isConcurrent() == true) { - synchronized (this.lock) { - if (data.containsKey(elementDataKey) == false) { - data.put(typedKey, dataFactory.create(this)); - } - } - } else { - data.put(typedKey, dataFactory.create(this)); - } + checkNotFinished(); + data.putIfAbsent(typedKey, dataFactory.create(this)); } return (T) data.get(typedKey); } @@ -304,16 +369,10 @@ public T getOrAdd( TypedKey untypedKey = new TypedKeyDefault<>( key.getName(), Object.class); + // No lock, as above. if (data.containsKey(key) == false) { - if (pipeline.isConcurrent() == true) { - synchronized (this.lock) { - if (data.containsKey(key) == false) { - data.put(key, dataFactory.create(this)); - } - } - } else { - data.put(key, dataFactory.create(this)); - } + checkNotFinished(); + data.putIfAbsent(key, dataFactory.create(this)); } try { result = data.get(key); @@ -328,7 +387,84 @@ public T getOrAdd( @Override public Map elementDataAsMap() { - return data.asStringKeyMap(); + return new ElementDataMap(data.asStringKeyMap()); + } + + /** + * The Element Data as a map, whose writes fail once processing has + * finished, whenever the map was taken. Reads go straight to the + * Flow Data's map, which gives copies of its keys, values and entries. + */ + private final class ElementDataMap extends AbstractMap { + private final Map live; + + ElementDataMap(Map live) { + this.live = live; + } + + @Override + public int size() { + return live.size(); + } + + @Override + public boolean isEmpty() { + return live.isEmpty(); + } + + @Override + public boolean containsKey(Object key) { + return live.containsKey(key); + } + + @Override + public boolean containsValue(Object value) { + return live.containsValue(value); + } + + @Override + public Object get(Object key) { + return live.get(key); + } + + @Override + public Set keySet() { + return live.keySet(); + } + + @Override + public Collection values() { + return live.values(); + } + + @Override + public Set> entrySet() { + return live.entrySet(); + } + + @Override + public Object put(String key, Object value) { + checkNotFinished(); + return live.put(key, value); + } + + @Override + public void putAll(Map m) { + checkNotFinished(); + live.putAll(m); + } + + @Override + public Object remove(Object key) { + checkNotFinished(); + return live.remove(key); + } + + @Override + public void clear() { + checkNotFinished(); + live.clear(); + } } @Override @@ -343,8 +479,21 @@ public Iterable elementDataAsIterable() { @Override @SuppressWarnings("rawtypes") public Map getWhere(PropertyMatcher matcher) { + if (processed.get() == false) { + throw new IllegalStateException( + "This instance has not yet been processed."); + } Map map = new HashMap<>(); for (FlowElement element : pipeline.getFlowElements()) { + // An element that did not run, for example because an earlier + // one stopped processing, has no data, and a property may have + // no value. Neither is the caller's mistake, so both are left + // out rather than thrown. See + // https://github.com/51Degrees/specifications/blob/706de1f0cb3edd560eb7fd7ef3bb19b93a4883dc/pipeline-specification/features/exception-handling.md#L57 + if (data.containsKey(element.getElementDataKey()) == false) { + continue; + } + ElementData elementData = get(element.getElementDataKey()); // We know the property has extends ElementPropertyMetaData as it // is a constraint of the FlowElement interface, so don't check this // cast @@ -352,10 +501,14 @@ public Map getWhere(PropertyMatcher matcher) { ElementPropertyMetaData property = (ElementPropertyMetaData)propertyObject; if (property.isAvailable() && matcher.isMatch(property)) { - map.put( - element.getElementDataKey() + "." + property.getName(), - get(element.getElementDataKey()) - .get(property.getName().toLowerCase()).toString()); + Object value = elementData.get( + property.getName().toLowerCase(Locale.ROOT)); + if (value != null) { + map.put( + element.getElementDataKey() + "." + + property.getName(), + value.toString()); + } } } } @@ -382,12 +535,27 @@ public EvidenceKeyFilter getEvidenceKeyFilter() { return pipeline.getEvidenceKeyFilter(); } + /** + * Close the Element Data that needs it and give pooled Element Data back + * to its pool. Only the first call does anything, because by a second + * call the pool may have given an instance to another request. + */ @Override public void close() throws Exception { + if (closed.getAndSet(true)) { + return; + } for (Object elementData : data.asStringKeyMap().values()) { if (AutoCloseable.class.isAssignableFrom(elementData.getClass())) { ((AutoCloseable) elementData).close(); } } + // Pooled Element Data goes back to its Flow Element's pool. A + // reference kept after this point is not supported. + for (Object elementData : data.asStringKeyMap().values()) { + if (elementData instanceof ElementDataBase) { + ((ElementDataBase) elementData).release(); + } + } } } diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowElementBase.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowElementBase.java index 3da07c441..fe713261c 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowElementBase.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowElementBase.java @@ -32,6 +32,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; /** * Provides a base class from which authors may create new FlowElements. The @@ -138,7 +140,21 @@ public void process(FlowData data) throws Exception { logger.debug("FlowElement '{}-{}' started processing.", getClass().getSimpleName(), hashCode()); startTime = System.currentTimeMillis(); } - processInternal(data); + // A Flow Element that shares an instance another has finished with + // opens it for writing again, and every Flow Element closes its own + // Element Data to writes once it has finished, even if it threw. + ElementDataBase own = ownElementData(data); + if (own != null) { + own.startWriting(); + } + try { + processInternal(data); + } finally { + own = ownElementData(data); + if (own != null) { + own.finishWriting(); + } + } logger.debug("FlowElement '{}}-{}' finished processing. Elapsed time: {}ms", getClass().getSimpleName(), hashCode(), (System.currentTimeMillis() - startTime)); @@ -146,6 +162,41 @@ public void process(FlowData data) throws Exception { } + /** + * True once getting the typed key has failed, as it does for a Flow + * Element with no Element Data of its own, so it is not tried again. + */ + private volatile boolean noOwnElementData = false; + + /** + * Get the Element Data this Flow Element writes, if the Flow Data holds + * it yet. + * @param data the Flow Data being processed + * @return the Element Data, or null if there is none + */ + private ElementDataBase ownElementData(FlowData data) { + if (noOwnElementData || this instanceof ParallelElements) { + return null; + } + TypedKey key; + try { + key = getTypedDataKey(); + } catch (RuntimeException e) { + noOwnElementData = true; + return null; + } + if (key == null) { + return null; + } + TryGetResult result = data.tryGetValue(key); + if (result != null && + result.hasValue() && + result.getValue() instanceof ElementDataBase) { + return (ElementDataBase) result.getValue(); + } + return null; + } + @Override public DataFactory getDataFactory() { if (dataFactory == null) { @@ -222,9 +273,24 @@ public void close() throws Exception { */ protected static class DataFactoryInternal implements DataFactory { + /** + * The most instances a pool keeps. More than run at once on most + * servers, so a busy server rarely allocates, and bounded so a burst + * does not hold memory for ever. + */ + static final int POOL_SIZE = + Math.max(64, Runtime.getRuntime().availableProcessors() * 8); + private final ElementDataFactory elementDataFactory; private final FlowElement element; + /** + * Element Data given back when its Flow Data was closed, ready to be + * given out again. Only instances that can be pooled go in. + */ + private final BlockingQueue pool = + new ArrayBlockingQueue<>(POOL_SIZE); + /** * Construct a new instance for the calling element. * @@ -240,7 +306,19 @@ protected static class DataFactoryInternal @Override public T create(FlowData flowData) { - return elementDataFactory.create(flowData, element); + T result = pool.poll(); + if (result != null) { + ((ElementDataBase) result).reuse(flowData); + } else { + result = elementDataFactory.create(flowData, element); + } + if (result instanceof ElementDataBase && + ((ElementDataBase) result).canBePooled()) { + final T pooled = result; + // A full pool drops the instance for the garbage collector. + ((ElementDataBase) result).setOnRelease(() -> pool.offer(pooled)); + } + return result; } } diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/ParallelElements.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/ParallelElements.java index b1a479b75..a5f899bb8 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/ParallelElements.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/ParallelElements.java @@ -32,10 +32,14 @@ /** * ParallelElements executes it's child {@link FlowElement} objects in parallel. + * A {@link Pipeline} lists a parallel group as one of these, so code that + * needs every Flow Element in use, such as the usage sharing element, reads + * the group's members with {@link #getFlowElements()}. The constructor is + * package-private, so only the Pipeline builders create one. * @see Specification */ @SuppressWarnings("rawtypes") -class ParallelElements +public class ParallelElements extends FlowElementBase { private final ExecutorService threadPool; @@ -59,7 +63,7 @@ class ParallelElements * Get an unmodifiable list of the child {@link FlowElement} instances. * @return child {@link FlowElement}s */ - List getFlowElements() { + public List getFlowElements() { return Collections.unmodifiableList(flowElements); } @@ -70,7 +74,8 @@ protected void processInternal(FlowData flowData) throws Exception { parallelCallers.add(new FlowElementCallable(element, flowData)); } try { - List> results = threadPool.invokeAll(parallelCallers); + List> results = + runMembers(flowData, parallelCallers); // all threads are now done for (Future future : results) { FlowError error; @@ -83,6 +88,29 @@ protected void processInternal(FlowData flowData) throws Exception { } } + /** + * Run every member and wait for them all to stop. While they run, the + * Flow Data adds the errors members record under its errors lock. + */ + private List> runMembers( + FlowData flowData, + List parallelCallers) + throws InterruptedException { + FlowDataDefault running = flowData instanceof FlowDataDefault ? + (FlowDataDefault) flowData : + null; + if (running != null) { + running.groupStarted(); + } + try { + return threadPool.invokeAll(parallelCallers); + } finally { + if (running != null) { + running.groupStopped(); + } + } + } + @Override public String getElementDataKey() { // This is because ParallelElements instances cannot translate diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/Pipeline.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/Pipeline.java index 098f54fee..f36061939 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/Pipeline.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/Pipeline.java @@ -61,6 +61,24 @@ public interface Pipeline extends AutoCloseable { */ boolean isConcurrent(); + /** + * True if exceptions that happen while processing are recorded in + * {@link FlowData#getErrors()} and logged, rather than thrown once + * processing is complete. A wrapper of the Pipeline, such as the web + * integration, can read it to decide whether to suppress and log an + * exception of its own. See the + * exception handling specification. + *

    + * The default gives the builder's default, so that an implementation + * written before this method was added still compiles. Override it to + * give the real setting. + * @return true if exceptions during processing are suppressed + */ + default boolean isSuppressProcessExceptions() { + return fiftyone.pipeline.core.Constants + .DEFAULT_SUPPRESS_PROCESS_EXCEPTIONS; + } + /** * Indicates whether the Pipeline has been closed using the {@link #close()} * method, either explicitly or as a result of a 'try with resource'. diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java index f10d49d1c..d177ba55c 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java @@ -206,7 +206,7 @@ public static Set> getAvailableElementBuilders() { config.setInputsFilter(new Predicate() { @Override public boolean test(String s) { - return s.toLowerCase().contains("flowelements"); + return s.toLowerCase(Locale.ROOT).contains("flowelements"); } }); config.setExpandSuperTypes(false); @@ -676,7 +676,7 @@ private List processBuildParameters( if (methods.size() == 0) { // If not then add the parameter to the list of parameters // to pass to the Build method instead. - buildParameterList.add(parameter.getKey().toLowerCase()); + buildParameterList.add(parameter.getKey().toLowerCase(Locale.ROOT)); } else { boolean methodCalled = false; int counter = 0; @@ -802,7 +802,7 @@ private List getMethods( Method[] methods) { int tries = 0; List matchingMethods = new ArrayList<>(); - String lowerMethodName = methodName.toLowerCase(); + String lowerMethodName = methodName.toLowerCase(Locale.ROOT); while (tries < 3 && matchingMethods.size() == 0) { List potentialMethods = null; @@ -812,7 +812,7 @@ private List getMethods( // First try and find a method that matches the // supplied name exactly. for (Method method : methods) { - if (method.getName().toLowerCase().equals(lowerMethodName)) { + if (method.getName().toLowerCase(Locale.ROOT).equals(lowerMethodName)) { matchingMethods.add(method); } } @@ -822,7 +822,7 @@ private List getMethods( // supplied name with 'set' added to the start. String tempName = "set" + lowerMethodName; for (Method method : methods) { - if (method.getName().toLowerCase().equals(tempName)) { + if (method.getName().toLowerCase(Locale.ROOT).equals(tempName)) { matchingMethods.add(method); } } @@ -835,10 +835,10 @@ private List getMethods( try { AlternateName alternateName = method.getAnnotation(AlternateName.class); - if (alternateName.value().toLowerCase().equals( + if (alternateName.value().toLowerCase(Locale.ROOT).equals( lowerMethodName)) { tempMethods.add(method); - } else if (alternateName.value().toLowerCase().equals( + } else if (alternateName.value().toLowerCase(Locale.ROOT).equals( "set" + lowerMethodName)) { tempMethods.add(method); } @@ -880,7 +880,7 @@ private List getMethods( private Class getBuilderType(String builderName) { int tries = 0; Class builderType = null; - String lowerBuilderName = builderName.toLowerCase(); + String lowerBuilderName = builderName.toLowerCase(Locale.ROOT); while (tries < 3 && builderType == null) { List> potentialBuilders = new ArrayList<>(); @@ -890,7 +890,7 @@ private Class getBuilderType(String builderName) { // First try and find a builder that matches the // supplied name exactly. for (Class builder : elementBuilders) { - if (builder.getSimpleName().toLowerCase().equals(lowerBuilderName)) { + if (builder.getSimpleName().toLowerCase(Locale.ROOT).equals(lowerBuilderName)) { potentialBuilders.add(builder); } } @@ -900,7 +900,7 @@ private Class getBuilderType(String builderName) { // Next, try and find a builder that matches the // supplied name with 'builder' added to the end. for (Class builder : elementBuilders) { - if (builder.getSimpleName().toLowerCase().equals(tempName)) { + if (builder.getSimpleName().toLowerCase(Locale.ROOT).equals(tempName)) { potentialBuilders.add(builder); } } @@ -912,7 +912,7 @@ private Class getBuilderType(String builderName) { for (Class builder : elementBuilders) { for (Annotation annotation : builder.getAnnotations()) { if (annotation instanceof ElementBuilder && - ((ElementBuilder) annotation).alternateName().toLowerCase().equals(lowerBuilderName)) { + ((ElementBuilder) annotation).alternateName().toLowerCase(Locale.ROOT).equals(lowerBuilderName)) { builders.add(builder); } } @@ -1004,6 +1004,20 @@ private Object parseToType( Class targetType, String value, String errorTextPrefix) { + // A configuration file gives an enumeration value as the name of the + // value, which is matched without regard to case. An enumeration + // with a parse method of its own is left to that method, which may + // accept more spellings and give its own message. + if (targetType.isEnum() && hasParseMethod(targetType) == false) { + for (Object constant : targetType.getEnumConstants()) { + if (((Enum) constant).name().equalsIgnoreCase(value.trim())) { + return constant; + } + } + throw new PipelineConfigurationException( + errorTextPrefix + ". '" + value + "' is not the name of " + + "one of its values."); + } // Check for a TryParse method on the type Method parse = null; Class nonPrimitiveType; @@ -1044,6 +1058,21 @@ private Object parseToType( return parsedValue; } + /** + * Whether the type has a method named 'parse...' that takes one string, + * as {@link #parseToType} looks for. + */ + private static boolean hasParseMethod(Class type) { + for (Method method : type.getMethods()) { + if (method.getName().startsWith("parse") && + method.getParameterTypes().length == 1 && + method.getParameterTypes()[0].equals(String.class)) { + return true; + } + } + return false; + } + /** * Evaluate strings for substitution of system properties etc. */ diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBase.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBase.java index 72cefd9a5..8df57efad 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBase.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBase.java @@ -22,6 +22,10 @@ package fiftyone.pipeline.core.flowelements; +import fiftyone.pipeline.annotations.AlternateName; +import fiftyone.pipeline.annotations.DefaultValue; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.Constants; import fiftyone.pipeline.core.configuration.PipelineOptions; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.data.factories.FlowDataFactory; @@ -67,9 +71,11 @@ public abstract class PipelineBuilderBase { */ protected final List services = new ArrayList<>(); - protected boolean autoCloseElements = false; + protected boolean autoCloseElements = + Constants.DEFAULT_AUTO_CLOSE_ELEMENTS; - protected boolean suppressProcessExceptions = false; + protected boolean suppressProcessExceptions = + Constants.DEFAULT_SUPPRESS_PROCESS_EXCEPTIONS; /** * Construct a new instance using the default {@link ILoggerFactory} @@ -102,6 +108,7 @@ protected List getFlowElements() { */ public Pipeline build() throws Exception { onPreBuild(); + checkRunsLast(flowElements); Pipeline pipeline = new PipelineDefault( loggerFactory.getLogger(Pipeline.class.getName()), flowElements, @@ -112,6 +119,58 @@ public Pipeline build() throws Exception { return pipeline; } + /** + * After a build has failed, close the Flow Elements this builder made + * itself and take them out of its list, because no caller holds them to + * close. A failure to close one is added to the build's error. + * @param made the Flow Elements this builder made + * @param error the error the build failed with + */ + protected void closeMadeElements( + List> made, + Exception error) { + for (FlowElement element : made) { + getFlowElements().remove(element); + try { + element.close(); + } catch (Exception e) { + error.addSuppressed(e); + } + } + } + + /** + * Refuse a Pipeline that puts a Flow Element that must run last, such as + * the usage sharing element, anywhere but last, or in a parallel group. + * @param elements the Flow Elements of the Pipeline, in order + * @throws PipelineConfigurationException if one is out of place + */ + @SuppressWarnings("rawtypes") + private static void checkRunsLast(List elements) { + for (int i = 0; i < elements.size(); i++) { + FlowElement element = elements.get(i); + if (element instanceof ParallelElements) { + for (FlowElement member : + ((ParallelElements) element).getFlowElements()) { + if (member instanceof RunsLast) { + throw runsLastRefused((RunsLast) member); + } + } + } else if (element instanceof RunsLast && + i != elements.size() - 1) { + throw runsLastRefused((RunsLast) element); + } + } + } + + private static PipelineConfigurationException runsLastRefused( + RunsLast element) { + return new PipelineConfigurationException( + element.getRunsLastName() + " must be the last Flow Element " + + "and outside any parallel group. Add it after every other " + + "Flow Element, and not as a member of a parallel group."); + } + /** * Add a {@link FlowElement} to the {@link Pipeline} to be run in series. * @param element to add @@ -178,11 +237,16 @@ protected void addServicesToPipeline(Pipeline pipeline) { /** * Configure the {@link Pipeline} to either call dispose on it's child * {@link FlowElement}s when it is disposed or not. + *

    + * A configuration file may also give this option as 'AutoDisposeElements' + * or 'SetAutoDisposeElements', the name other languages use. * @param autoClose true if the Pipeline should call dispose on it's child * elements when it is disposed * @return this builder */ @SuppressWarnings("unchecked") + @DefaultValue(booleanValue = Constants.DEFAULT_AUTO_CLOSE_ELEMENTS) + @AlternateName("SetAutoDisposeElements") public T setAutoCloseElements(boolean autoClose) { this.autoCloseElements = autoClose; return (T) this; @@ -198,6 +262,7 @@ public T setAutoCloseElements(boolean autoClose) { * @return this builder */ @SuppressWarnings("unchecked") + @DefaultValue(booleanValue = Constants.DEFAULT_SUPPRESS_PROCESS_EXCEPTIONS) public T setSuppressProcessException(boolean suppress) { this.suppressProcessExceptions = suppress; return (T) this; diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java index 302b51d94..bc9ee45d3 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java @@ -109,11 +109,11 @@ class PipelineDefault implements PipelineInternal { /** * A filter that will only include the evidence keys that can be used by at - * least one {@link FlowElement} within this pipeline. - * (Will only be populated after the {@link #getEvidenceKeyFilter()} method - * is called.) + * least one {@link FlowElement} within this pipeline. Built once in the + * constructor, because the elements do not change after that, and final + * so every request can read it with no lock. */ - private EvidenceKeyFilter evidenceKeyFilter = null; + private final EvidenceKeyFilter evidenceKeyFilter; /** * The pipeline maintains a map of the elements it contains indexed by type. @@ -150,7 +150,11 @@ class PipelineDefault implements PipelineInternal { boolean suppressProcessExceptions) { this.logger = logger; this.flowDataFactory = flowDataFactory; - this.flowElements = flowElements; + // A copy, so that the builder adding to its own list later cannot + // change this Pipeline, which is immutable. See + // https://github.com/51Degrees/specifications/blob/706de1f0cb3edd560eb7fd7ef3bb19b93a4883dc/pipeline-specification/conceptual-overview.md#L118 + this.flowElements = Collections.unmodifiableList( + new ArrayList<>(flowElements)); this.services = new ArrayList<>(); this.autoDisposeElements = autoDisposeElements; this.suppressProcessExceptions = suppressProcessExceptions; @@ -161,6 +165,13 @@ class PipelineDefault implements PipelineInternal { for (FlowElement element : this.flowElements) { isConcurrent = isConcurrent || element.isConcurrent(); } + + EvidenceKeyFilterAggregator filter = new EvidenceKeyFilterAggregator(); + for (FlowElement element : this.flowElements) { + filter.addFilter(element.getEvidenceKeyFilter()); + } + evidenceKeyFilter = filter; + getElementAvailableProperties(); logger.debug("Pipeline '" + hashCode() + "' created."); @@ -192,18 +203,6 @@ public FlowData createFlowData() throws RuntimeException { @Override public EvidenceKeyFilter getEvidenceKeyFilter() { - if (evidenceKeyFilter == null) { - synchronized (lock) { - if (evidenceKeyFilter == null) { - EvidenceKeyFilterAggregator filter = - new EvidenceKeyFilterAggregator(); - for (FlowElement element : flowElements) { - filter.addFilter(element.getEvidenceKeyFilter()); - } - evidenceKeyFilter = filter; - } - } - } return evidenceKeyFilter; } @@ -212,6 +211,11 @@ public boolean isConcurrent() { return isConcurrent; } + @Override + public boolean isSuppressProcessExceptions() { + return suppressProcessExceptions; + } + @Override public boolean isClosed() { return isClosed; diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/RunsLast.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/RunsLast.java new file mode 100644 index 000000000..9486d0f69 --- /dev/null +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/RunsLast.java @@ -0,0 +1,41 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +/** + * Implemented by a {@link FlowElement} that must be the last Flow Element of + * a {@link Pipeline} and outside any parallel group, such as the usage + * sharing element, which reads what every other Flow Element has added to + * the Evidence. Building a Pipeline that puts one anywhere else fails with a + * {@link fiftyone.pipeline.core.exceptions.PipelineConfigurationException}. + * @see Specification + */ +public interface RunsLast { + + /** + * The name the error gives the element when a Pipeline puts it anywhere + * but last, such as "The usage sharing element". + * @return the name of the element + */ + String getRunsLastName(); +} diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/typed/TypedKeyMap.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/typed/TypedKeyMap.java index 403c410f2..c9ef59db3 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/typed/TypedKeyMap.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/typed/TypedKeyMap.java @@ -62,6 +62,23 @@ public interface TypedKeyMap extends Closeable { */ void put(TypedKey typedKey, T value); + /** + * Add a value unless one with the same name is already there, and return + * whichever value the map then holds for the name. Implementations that + * several threads add to do it without a lock. + * + * @param typedKey the key to add + * @param value the value to add if the name is not there yet + * @param the type of the value + * @return the value now held for the name + */ + default T putIfAbsent(TypedKey typedKey, T value) { + if (containsKey(typedKey.getName()) == false) { + put(typedKey, value); + } + return get(typedKey); + } + T removeIfExists(TypedKey typedKey); boolean containsKey(String key); diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/typed/TypedKeyMapBuilder.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/typed/TypedKeyMapBuilder.java index 72e87bd3d..e5a2b0caa 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/typed/TypedKeyMapBuilder.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/typed/TypedKeyMapBuilder.java @@ -27,6 +27,7 @@ import java.io.Closeable; import java.util.*; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.logging.Level; import java.util.logging.Logger; @@ -78,14 +79,11 @@ public TypedKeyMap build() { * Extends the internal implementation to add thread-safety to the * {@link #get(Object)} and {@link #put(String, Object)} methods. */ + /** + * Adds with compare and swap, so members of a parallel group add without + * a lock, and reads take none. Removal is rare and keeps its lock. + */ private static class TypedKeyMapConcurrent extends TypedKeyMapInternal { - @Override - public void put(TypedKey typedKey, T value) { - synchronized (this) { - super.put(typedKey, value); - } - } - @Override public T removeIfExists(TypedKey typedKey) { synchronized (this) { @@ -101,16 +99,56 @@ public T removeIfExists(TypedKey typedKey) { private static class TypedKeyMapInternal implements TypedKeyMap, Map, Closeable { private static final Entry[] emptyEntryPair = new Entry[]{null, null}; - private Entry _head = null; + + /** + * The newest entry. A new entry is linked to the current head and + * swapped in with one compare and swap, retrying if another thread + * added first, so readers follow the links with no lock. + */ + private volatile Entry _head = null; + + private static final AtomicReferenceFieldUpdater + HEAD = AtomicReferenceFieldUpdater.newUpdater( + TypedKeyMapInternal.class, Entry.class, "_head"); + + /** + * Find the entry for a name in the list starting at head. + */ + private static Entry findFrom(Entry head, String typedKeyName) { + Entry current = head; + while (current != null) { + if (current._key.getName().equalsIgnoreCase(typedKeyName)) { + return current; + } + current = current._next; + } + return null; + } + + @Override + public T putIfAbsent(TypedKey typedKey, T value) { + while (true) { + Entry head = _head; + Entry existing = findFrom(head, typedKey.getName()); + if (existing != null) { + return typedKey.getType().cast(existing._value); + } + Entry first = new Entry<>(typedKey, value); + first._next = head; + if (HEAD.compareAndSet(this, head, first)) { + return value; + } + } + } private Entry[] findWithPrevious(String typedKeyName) { Entry current = _head; if (current != null) { - if (current._key.getName().equals(typedKeyName)) { + if (current._key.getName().equalsIgnoreCase(typedKeyName)) { return new Entry[]{null, current}; } while (current._next != null) { - if (current._next._key.getName().equals(typedKeyName)) { + if (current._next._key.getName().equalsIgnoreCase(typedKeyName)) { return new Entry[]{current, current._next}; } current = current._next; @@ -223,10 +261,11 @@ public void put(TypedKey typedKey, T value) { current._value = value; } else { // The key does not exist so add it to the head of the linked - // list. - Entry first = new Entry<>(typedKey, value); - first._next = _head; - _head = first; + // list, or overwrite the value another thread added first. + T held = putIfAbsent(typedKey, value); + if (held != value) { + put(typedKey, value); + } } } @@ -364,8 +403,8 @@ public void close() { private static class Entry { final TypedKey _key; - T _value; - Entry _next = null; + volatile T _value; + volatile Entry _next = null; Entry(TypedKey key, T value) { _key = key; diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/configuration/ConfigurationFileLanguagesTest.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/configuration/ConfigurationFileLanguagesTest.java new file mode 100644 index 000000000..049c2b8bb --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/configuration/ConfigurationFileLanguagesTest.java @@ -0,0 +1,73 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.configuration; + +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertEquals; + +/** + * A configuration file written with the structure and names the .NET sample + * configuration file uses (PipelineOptions, Elements, Element, BuilderName, + * BuildParameters, a builder name ending 'Builder' and builder method names + * starting 'Set', at element and Pipeline level) is read and builds a + * Pipeline. + *

    + * Specification: pipeline-specification/features/pipeline-configuration.md + * line 40 at 51Degrees/specifications 706de1f0, "The configuration file + * definition MUST be as similar as possible to configuration files for other + * languages." + */ +public class ConfigurationFileLanguagesTest { + + @Test + public void dotNetNames_AreRead() throws Exception { + PipelineOptions options = PipelineOptionsFactory.getOptionsFromFile( + getFilePath("options-dotnet-names.xml")); + + assertEquals(1, options.elements.size()); + assertEquals( + "MultiplyByElementBuilder", + options.elements.get(0).builderName); + assertEquals("8", options.elements.get(0).buildParameters.get("multiple")); + assertEquals( + "true", + options.pipelineBuilderParameters.get("SetAutoDisposeElements")); + } + + @Test + public void dotNetNames_BuildAPipeline() throws Exception { + PipelineOptions options = PipelineOptionsFactory.getOptionsFromFile( + getFilePath("options-dotnet-names.xml")); + + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .buildFromConfiguration(options)) { + assertEquals(1, pipeline.getFlowElements().size()); + } + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/data/EvidenceKeyFilterAggregatorTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/data/EvidenceKeyFilterAggregatorTests.java index f13534892..f88b3c60b 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/data/EvidenceKeyFilterAggregatorTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/data/EvidenceKeyFilterAggregatorTests.java @@ -25,6 +25,7 @@ import org.junit.Test; import java.util.Arrays; +import java.util.Comparator; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -76,15 +77,15 @@ public void EvidenceKeyFilterAggregator_Include_Aggregated() { } /** - * Check that aggregated case-sensitive and case-insensitive filters - * still work as expected. + * Check that an aggregated case-insensitive filter, and one given a + * comparator that compares with case, still work as expected. */ @Test public void EvidenceKeyFilterAggregator_CaseSensitivity() { EvidenceKeyFilter filter1 = new EvidenceKeyFilterWhitelist( Arrays.asList("key1"), String.CASE_INSENSITIVE_ORDER); EvidenceKeyFilter filter2 = new EvidenceKeyFilterWhitelist( - Arrays.asList("key2")); + Arrays.asList("key2"), Comparator.naturalOrder()); EvidenceKeyFilterAggregator filter = new EvidenceKeyFilterAggregator(); filter.addFilter(filter1); diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/data/EvidenceKeyFilterWhitelistTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/data/EvidenceKeyFilterWhitelistTests.java index b37156162..3378e7f41 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/data/EvidenceKeyFilterWhitelistTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/data/EvidenceKeyFilterWhitelistTests.java @@ -25,6 +25,7 @@ import org.junit.Test; import java.util.Arrays; +import java.util.HashMap; import java.util.Map; import static org.junit.Assert.*; @@ -43,6 +44,27 @@ public void EvidenceKeyFilterWhitelist_Include_CheckKeys() { assertFalse(filter.include("key3")); } + /** + * Evidence keys ignore case, so a whitelist made without a comparator + * ignores case too, whether it is made from a list or from a map. + */ + @Test + public void EvidenceKeyFilterWhitelist_Default_IgnoresCase() { + filter = new EvidenceKeyFilterWhitelist( + Arrays.asList("header.user-agent")); + + assertTrue(filter.include("Header.User-Agent")); + assertTrue(filter.include("HEADER.USER-AGENT")); + assertFalse(filter.include("header.user-agent2")); + + Map keys = new HashMap<>(); + keys.put("query.Session-Id", 2); + filter = new EvidenceKeyFilterWhitelist(keys); + + assertTrue(filter.include("query.session-id")); + assertEquals(Integer.valueOf(2), filter.order("QUERY.SESSION-ID")); + } + @Test public void EvidenceKeyFilterWhitelist_List() { filter = new EvidenceKeyFilterWhitelist( diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/AcceptedEvidenceDeclaredTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/AcceptedEvidenceDeclaredTests.java new file mode 100644 index 000000000..b8f7b6aba --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/AcceptedEvidenceDeclaredTests.java @@ -0,0 +1,59 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Every Flow Element declares the Evidence it accepts, because the base + * Flow Element leaves getEvidenceKeyFilter abstract, so the compiler makes + * each concrete element give one. + */ +public class AcceptedEvidenceDeclaredTests { + + @Test + public void FlowElement_DeclaresTheEvidenceFilter() throws Exception { + Method method = FlowElement.class.getMethod("getEvidenceKeyFilter"); + assertEquals(EvidenceKeyFilter.class, method.getReturnType()); + assertTrue("The interface gives a default, so an element need not " + + "declare the Evidence it accepts.", + method.isDefault() == false); + } + + @Test + public void FlowElementBase_LeavesTheEvidenceFilterAbstract() + throws Exception { + Method method = FlowElementBase.class.getDeclaredMethod( + "getEvidenceKeyFilter"); + assertTrue("FlowElementBase gives a default, so an element need " + + "not declare the Evidence it accepts.", + Modifier.isAbstract(method.getModifiers())); + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/BuilderDefaultValueTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/BuilderDefaultValueTests.java new file mode 100644 index 000000000..f9159208e --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/BuilderDefaultValueTests.java @@ -0,0 +1,73 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.annotations.DefaultValue; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Every configuration setter of the Pipeline builder gives its default + * value beside it, with {@link DefaultValue}. See + * reference-implementation-notes.md line 79: + * "In all cases, there SHOULD be some mechanism to specify the default + * values alongside configuration property setters." + */ +public class BuilderDefaultValueTests { + + /** + * The public setters declared by the builders that do not carry a + * {@link DefaultValue}. + */ + static List missingDefaults(Class... builders) { + List missing = new ArrayList<>(); + for (Class builder : builders) { + for (Method method : builder.getDeclaredMethods()) { + if (Modifier.isPublic(method.getModifiers()) && + method.getName().startsWith("set") && + method.isSynthetic() == false && + method.isBridge() == false && + method.getAnnotation(DefaultValue.class) == null) { + missing.add(builder.getSimpleName() + "#" + + method.getName() + + Arrays.toString(method.getParameterTypes())); + } + } + } + return missing; + } + + @Test + public void pipelineBuilderSetters_GiveTheirDefaults() { + assertEquals(new ArrayList(), missingDefaults( + PipelineBuilderBase.class, + PipelineBuilder.class)); + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/CacheKeyTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/CacheKeyTests.java new file mode 100644 index 000000000..469dda135 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/CacheKeyTests.java @@ -0,0 +1,109 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.DataKey; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByFiveElement; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.mockito.Mockito.mock; + +/** + * Cache keys are built from the Evidence that the element advertises, with + * values compared with their case. See the Generation of keys section of + * caching.md. + */ +public class CacheKeyTests { + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + private final MultiplyByFiveElement element = + new MultiplyByFiveElement(mock(Logger.class)); + + private DataKey keyFor(Map evidence) throws Exception { + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(element) + .build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence(evidence); + EvidenceKeyFilter filter = element.getEvidenceKeyFilter(); + return flowData.generateKey(filter); + } + } + + private static Map evidence(Object... pairs) { + Map map = new java.util.LinkedHashMap<>(); + for (int i = 0; i < pairs.length; i += 2) { + map.put((String) pairs[i], pairs[i + 1]); + } + return map; + } + + /** + * caching.md line 59 at 706de1f0: "All Flow Elements MUST advertize the + * Evidence keys that they make use of." The key is built from those keys + * alone, so Evidence the element does not use does not change it, and + * Evidence it uses does. + */ + @Test + public void key_IsBuiltFromTheAdvertisedEvidenceOnly() throws Exception { + String used = element.evidenceKeys.get(0); + DataKey plain = keyFor(evidence(used, 2)); + DataKey withOther = keyFor(evidence(used, 2, "query.other", "x")); + DataKey otherValue = keyFor(evidence(used, 3)); + + assertEquals( + "Evidence the element does not use changed the key.", + plain, withOther); + assertEquals(plain.hashCode(), withOther.hashCode()); + assertNotEquals( + "Evidence the element uses did not change the key.", + plain, otherValue); + } + + /** + * caching.md line 82 at 706de1f0: "Comparison of Evidence values MUST + * be case-sensitive." + */ + @Test + public void values_AreComparedWithCase() throws Exception { + String used = element.evidenceKeys.get(0); + assertNotEquals( + "Two values that differ only in case gave the same key.", + keyFor(evidence(used, "abc")), + keyFor(evidence(used, "ABC"))); + assertEquals( + keyFor(evidence(used, "abc")), + keyFor(evidence(used, "abc"))); + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ElementCreationConsistencyTest.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ElementCreationConsistencyTest.java new file mode 100644 index 000000000..9c334c03f --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ElementCreationConsistencyTest.java @@ -0,0 +1,141 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertTrue; + +/** + * Every Flow Element in the repository is made the same way, by a builder + * class (pipeline-specification/usage-examples.md, "Creating Flow Elements + * SHOULD always be done using a consistent mechanism. In the case of C#, we + * use a separate builder class."). + * + * The test reads the main source of every module below the repository + * root. A concrete class that extends one of the Flow Element base classes + * is a Flow Element, unless its name ends in "Base", which marks a base + * class that released code may still make directly. Each Flow Element must + * be made by a class whose name ends in "Builder", and no other main code + * may make one with {@code new}. + */ +public class ElementCreationConsistencyTest { + + /** + * The base classes a concrete Flow Element extends. + */ + private static final Pattern ELEMENT_BASE = Pattern.compile( + "\\bextends\\s+(FlowElementBase|AspectEngineBase|" + + "OnPremiseAspectEngineBase|FiftyOneOnPremiseAspectEngineBase|" + + "CloudAspectEngineBase|PropertyKeyedCloudEngineBase|" + + "ShareUsageBase|TranslationEngineBase)\\b"); + + private static final Pattern CLASS = Pattern.compile( + "^public\\s+((?:abstract\\s+|final\\s+)*)class\\s+(\\w+)([^{]*)\\{", + Pattern.MULTILINE); + + private static Path root() { + String root = System.getProperty("project.root"); + return Paths.get(root == null ? "../.." : root).toAbsolutePath() + .normalize(); + } + + /** + * The text of each main source file, by its path. + */ + private static Map mainSources() throws IOException { + Map sources = new HashMap<>(); + try (Stream files = Files.walk(root())) { + for (Path file : files + .filter(p -> p.toString().endsWith(".java")) + .filter(p -> p.toString().replace(File.separatorChar, '/') + .contains("/src/main/java/")) + .filter(p -> p.toString().replace(File.separatorChar, '/') + .contains("/target/") == false) + .collect(Collectors.toList())) { + sources.put(file, new String(Files.readAllBytes(file), + StandardCharsets.UTF_8)); + } + } + return sources; + } + + private static boolean isBuilder(Path file) { + String name = file.getFileName().toString(); + return name.endsWith("Builder.java") || + name.endsWith("BuilderBase.java"); + } + + @Test + public void everyElementIsMadeByItsBuilderAlone() throws IOException { + Map sources = mainSources(); + List elements = new ArrayList<>(); + for (String source : sources.values()) { + Matcher matcher = CLASS.matcher(source); + if (matcher.find() && matcher.group(1).contains("abstract") == false && + matcher.group(2).endsWith("Base") == false && + ELEMENT_BASE.matcher(matcher.group(3)).find()) { + elements.add(matcher.group(2)); + } + } + assertTrue("No Flow Elements found below " + root(), + elements.size() > 10); + + List problems = new ArrayList<>(); + for (String element : elements) { + Pattern made = Pattern.compile("\\bnew\\s+" + element + "\\s*[(<]"); + boolean byBuilder = false; + for (Map.Entry source : sources.entrySet()) { + if (made.matcher(source.getValue()).find()) { + if (isBuilder(source.getKey())) { + byBuilder = true; + } else { + problems.add(element + " is made outside a builder, in " + + root().relativize(source.getKey())); + } + } + } + if (byBuilder == false) { + problems.add(element + " is made by no builder"); + } + } + Collections.sort(problems); + assertTrue(String.join("\n", problems), problems.isEmpty()); + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ElementDataKeyCaseTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ElementDataKeyCaseTests.java new file mode 100644 index 000000000..c397b1001 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ElementDataKeyCaseTests.java @@ -0,0 +1,123 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.TryGetResult; +import fiftyone.pipeline.core.testclasses.data.TestElementData; +import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.core.typed.TypedKeyDefault; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; + +import java.util.Locale; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * The string key of an Element Data instance is compared without regard to + * case. + *

    + * Specification: pipeline-specification/features/access-to-results.md line 11 + * at 51Degrees/specifications 706de1f0, "String comparisons MUST be case + * insensitive." + */ +public class ElementDataKeyCaseTests { + + private FlowDataDefault flowData; + private ElementData data; + + @Before + public void init() { + flowData = new FlowDataDefault( + mock(Logger.class), + mock(PipelineInternal.class), + new EvidenceDefault(mock(Logger.class))); + // Element Data is added before processing, because a Flow Data + // that has finished processing refuses new Element Data. Each test + // processes once its own adds are done. + data = flowData.getOrAdd( + "device", + (FlowElement.DataFactory) fd -> + new TestElementData(mock(Logger.class), fd)); + } + + @After + public void close() throws Exception { + flowData.close(); + } + + @Test + public void get_StringKeyInAnotherCase_ReturnsTheData() { + flowData.process(); + + assertSame(data, flowData.get("Device")); + assertSame(data, flowData.get("DEVICE")); + } + + @Test + public void get_TypedKeyInAnotherCase_ReturnsTheData() { + flowData.process(); + TypedKey key = new TypedKeyDefault<>( + "Device", + TestElementData.class); + + assertSame(data, flowData.get(key)); + TryGetResult result = flowData.tryGetValue(key); + assertTrue(result.hasValue()); + assertSame(data, result.getValue()); + } + + @Test + public void getOrAdd_KeyInAnotherCase_KeepsOneInstance() { + ElementData again = flowData.getOrAdd( + "DEVICE", + (FlowElement.DataFactory) fd -> + new TestElementData(mock(Logger.class), fd)); + flowData.process(); + + assertSame(data, again); + assertEquals(1, flowData.elementDataAsMap().size()); + } + + @Test + public void get_KeyInAnotherCase_TurkishLocale_ReturnsTheData() { + Locale original = Locale.getDefault(); + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + try { + ElementData ip = flowData.getOrAdd( + "ip", + (FlowElement.DataFactory) fd -> + new TestElementData(mock(Logger.class), fd)); + flowData.process(); + assertSame(ip, flowData.get("IP")); + } finally { + Locale.setDefault(original); + } + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ElementDataPoolTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ElementDataPoolTests.java new file mode 100644 index 000000000..b6a5f37a2 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ElementDataPoolTests.java @@ -0,0 +1,343 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.ElementDataBase; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * Element Data that can be pooled is given out again once the Flow Data that + * held it is closed, so a Flow Element allocates one instance for each + * request running at once, not one for every request. A pooled instance + * holds nothing of the request it served before. + * @see Specification + */ +public class ElementDataPoolTests { + + private static final Logger LOGGER = + LoggerFactory.getLogger(ElementDataPoolTests.class); + + private static final int REQUESTS = 1000; + + private static final int THREADS = 8; + + private static final long WAIT_SECONDS = 60; + + /** + * Element Data whose only state is its map, so it can be pooled. + */ + public static class PooledData extends ElementDataBase { + public PooledData(Logger logger, FlowData flowData) { + super(logger, flowData); + } + + protected boolean isPoolable() { + return true; + } + } + + /** + * Counts the Element Data it allocates, and writes the request's value + * into its own, with a mark on the first request only. + */ + private static class Counter + extends FlowElementBase { + private final AtomicInteger allocated; + + Counter(AtomicInteger allocated) { + super(LOGGER, (flowData, element) -> { + allocated.incrementAndGet(); + return new PooledData(LOGGER, flowData); + }); + this.allocated = allocated; + } + + @Override + protected void processInternal(FlowData data) { + PooledData own = data.getOrAdd(getTypedDataKey(), getDataFactory()); + Object request = data.getEvidence().get("query.request"); + assertNull("A pooled instance kept the last request's value", + own.get("request")); + own.put("request", request); + if ("0".equals(request)) { + own.put("first", "only"); + } + } + + @Override + public String getElementDataKey() { + return "counter"; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist( + Collections.singletonList("query.request")); + } + + @Override + public List getProperties() { + return Collections.emptyList(); + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + } + + /** + * Element Data that can be pooled and that a lazy task writes, until the + * task the test gives it has finished. + */ + public static class TaskData extends ElementDataBase { + private volatile CompletableFuture task = null; + + public TaskData(Logger logger, FlowData flowData) { + super(logger, flowData); + } + + protected boolean isPoolable() { + return true; + } + + protected boolean hasLazyTaskRunning() { + return task != null && task.isDone() == false; + } + } + + /** + * Gives its Element Data the next task the test sets. + */ + private static class TaskStarter + extends FlowElementBase { + private volatile CompletableFuture next = null; + + TaskStarter() { + super(LOGGER, + (flowData, element) -> new TaskData(LOGGER, flowData)); + } + + @Override + protected void processInternal(FlowData data) { + data.getOrAdd(getTypedDataKey(), getDataFactory()).task = next; + } + + @Override + public String getElementDataKey() { + return "task"; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist(Collections.emptyList()); + } + + @Override + public List getProperties() { + return Collections.emptyList(); + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + } + + private static TaskData process(Pipeline pipeline) throws Exception { + try (FlowData data = pipeline.createFlowData()) { + data.process(); + return (TaskData) data.get("task"); + } + } + + private static void request(Pipeline pipeline, int i) throws Exception { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.request", Integer.toString(i)); + data.process(); + ElementDataBase own = (ElementDataBase) data.get("counter"); + assertEquals(Integer.toString(i), own.get("request")); + if (i != 0) { + assertNull("A pooled instance kept the first request's mark", + own.get("first")); + } + } + } + + @Test + public void oneThread_AllocatesOneInstance() throws Exception { + AtomicInteger allocated = new AtomicInteger(); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElement(new Counter(allocated)) + .setAutoCloseElements(true) + .build()) { + for (int i = 0; i < REQUESTS; i++) { + request(pipeline, i); + } + } + assertEquals( + "Element Data allocated for " + REQUESTS + " requests one after " + + "another", + 1, + allocated.get()); + } + + @Test + public void manyThreads_AllocatesNoMoreThanRunAtOnce() throws Exception { + AtomicInteger allocated = new AtomicInteger(); + ExecutorService threads = Executors.newFixedThreadPool(THREADS); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElement(new Counter(allocated)) + .setAutoCloseElements(true) + .build()) { + List> done = new ArrayList<>(); + for (int t = 0; t < THREADS; t++) { + final int first = t * REQUESTS + 1; + done.add(threads.submit(() -> { + for (int i = first; i < first + REQUESTS; i++) { + request(pipeline, i); + } + return null; + })); + } + for (Future each : done) { + each.get(WAIT_SECONDS, TimeUnit.SECONDS); + } + } finally { + threads.shutdownNow(); + } + assertTrue( + "Element Data allocated for " + (THREADS * REQUESTS) + + " requests on " + THREADS + " threads: " + allocated.get(), + allocated.get() <= THREADS); + } + + @Test + public void runningLazyTask_KeepsTheInstanceFromItsPool() throws Exception { + TaskStarter starter = new TaskStarter(); + CompletableFuture running = new CompletableFuture<>(); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElement(starter) + .setAutoCloseElements(true) + .build()) { + starter.next = running; + TaskData first = process(pipeline); + + starter.next = CompletableFuture.completedFuture(null); + TaskData second = process(pipeline); + assertNotSame( + "An instance went back to its pool while its lazy task ran", + first, + second); + + TaskData third = process(pipeline); + assertSame( + "An instance whose lazy task had finished was not pooled", + second, + third); + } finally { + running.complete(null); + } + } + + /** + * Closing a Flow Data a second time, as a try-with-resources around one + * that a web filter also closes does, must not give back an instance + * that the pool has since given to another request. + */ + @Test + public void closingTwice_GivesNoInstanceToTwoRequests() throws Exception { + AtomicInteger allocated = new AtomicInteger(); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElement(new Counter(allocated)) + .setAutoCloseElements(true) + .build(); + FlowData first = pipeline.createFlowData()) { + first.addEvidence("query.request", "first"); + first.process(); + Object pooled = first.get("counter"); + first.close(); + + try (FlowData second = pipeline.createFlowData()) { + second.addEvidence("query.request", "second"); + second.process(); + assertSame( + "The instance was not reused, so this proves nothing", + pooled, + second.get("counter")); + + first.close(); + + try (FlowData third = pipeline.createFlowData()) { + third.addEvidence("query.request", "third"); + third.process(); + assertNotSame( + "A second close gave an instance that another " + + "request holds back to its pool", + pooled, + third.get("counter")); + } + assertEquals( + "The second request's value was overwritten", + "second", + ((ElementDataBase) second.get("counter")).get("request")); + } + } + assertEquals( + "Element Data allocated for three requests, two at once", + 2, + allocated.get()); + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/EveryFlowElementCheckedTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/EveryFlowElementCheckedTests.java new file mode 100644 index 000000000..95774ec24 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/EveryFlowElementCheckedTests.java @@ -0,0 +1,255 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +/** + * Every Flow Element in the repository is processed by a check that then + * writes to its Element Data and expects the error. Each module that holds + * Flow Elements has a test class named {@value #CHECK}, which names each of + * the module's Flow Elements. This test finds every class in the + * repository's main source that is not abstract and extends + * {@link FlowElementBase}, directly or through other classes, and fails for + * one that its module's check does not name. + * @see Specification + */ +public class EveryFlowElementCheckedTests { + + static final String CHECK = "WritesAfterProcessingTests.java"; + + private static final List TOP_FOLDERS = Arrays.asList( + "pipeline", "device-detection", "ip-intelligence", "location", + "examples"); + + private static final Set SKIPPED_FOLDERS = new HashSet<>( + Arrays.asList("target", "cxx", "node_modules", ".git")); + + private static final Pattern CLASS = Pattern.compile( + "(?m)^[\\t ]*((?:public|protected|private|abstract|final|static)" + + "\\s+)*class\\s+(\\w+)"); + + private static final Pattern EXTENDS = Pattern.compile( + "\\bextends\\s+([\\w.]+)"); + + private static class Found { + final String name; + final String parent; + final boolean isAbstract; + final Path module; + + Found(String name, String parent, boolean isAbstract, Path module) { + this.name = name; + this.parent = parent; + this.isAbstract = isAbstract; + this.module = module; + } + } + + /** + * The repository root, found from the module's folder, or null when the + * module is built outside the repository. + */ + private static Path root() { + Path dir = Paths.get("").toAbsolutePath(); + while (dir != null) { + if (Files.isDirectory(dir.resolve("pipeline") + .resolve("pipeline.core")) && + Files.isDirectory(dir.resolve("device-detection"))) { + return dir; + } + dir = dir.getParent(); + } + return null; + } + + private static List javaFiles(Path root) throws IOException { + final List files = new ArrayList<>(); + for (String top : TOP_FOLDERS) { + Path folder = root.resolve(top); + if (Files.isDirectory(folder) == false) { + continue; + } + Files.walkFileTree(folder, new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory( + Path dir, + BasicFileAttributes attributes) { + return SKIPPED_FOLDERS.contains( + String.valueOf(dir.getFileName())) ? + FileVisitResult.SKIP_SUBTREE : + FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile( + Path file, + BasicFileAttributes attributes) { + if (file.toString().endsWith(".java")) { + files.add(file); + } + return FileVisitResult.CONTINUE; + } + }); + } + return files; + } + + /** + * The module folder a source file belongs to, being the folder that + * holds its src folder. + */ + private static Path module(Path file) { + for (Path dir = file.getParent(); dir != null; dir = dir.getParent()) { + if (String.valueOf(dir.getFileName()).equals("src")) { + return dir.getParent(); + } + } + return null; + } + + private static String read(Path file) throws IOException { + return new String(Files.readAllBytes(file), StandardCharsets.UTF_8); + } + + /** + * Drop type arguments, so that an extends inside them is not read as + * the class's own. + */ + private static String withoutTypeArguments(String text) { + StringBuilder result = new StringBuilder(); + int depth = 0; + for (char c : text.toCharArray()) { + if (c == '<') { + depth++; + } else if (c == '>') { + depth--; + } else if (depth == 0) { + result.append(c); + } + } + return result.toString(); + } + + private static boolean isFlowElement( + String name, + Map classes) { + Set seen = new HashSet<>(); + String current = name; + while (current != null && seen.add(current)) { + if (current.equals("FlowElementBase")) { + return true; + } + Found found = classes.get(current); + current = found == null ? null : found.parent; + } + return false; + } + + @Test + public void everyFlowElement_HasItsModulesCheck() throws IOException { + Path root = root(); + assumeTrue("Not built inside the repository", root != null); + + Map classes = new HashMap<>(); + List mainClasses = new ArrayList<>(); + Map checks = new HashMap<>(); + for (Path file : javaFiles(root)) { + String path = file.toString().replace('\\', '/'); + String text = read(file); + if (path.contains("/src/test/") && + file.getFileName().toString().equals(CHECK)) { + checks.put(module(file), text); + } + if (path.contains("/src/main/java/") == false) { + continue; + } + Matcher matcher = CLASS.matcher(text); + while (matcher.find()) { + int end = text.indexOf('{', matcher.end()); + if (end < 0) { + continue; + } + String declaration = withoutTypeArguments( + text.substring(matcher.end(), end)); + Matcher parent = EXTENDS.matcher(declaration); + String parentName = null; + if (parent.find()) { + String[] parts = parent.group(1).split("\\."); + parentName = parts[parts.length - 1]; + } + Found found = new Found( + matcher.group(2), + parentName, + matcher.group(0).contains("abstract"), + module(file)); + classes.put(found.name, found); + mainClasses.add(found); + } + } + + List missing = new ArrayList<>(); + int flowElements = 0; + for (Found found : mainClasses) { + if (found.isAbstract || + isFlowElement(found.name, classes) == false) { + continue; + } + flowElements++; + String check = checks.get(found.module); + if (check == null || + Pattern.compile("\\b" + found.name + "\\b") + .matcher(check).find() == false) { + missing.add(found.name + " in " + + root.relativize(found.module).toString() + .replace('\\', '/')); + } + } + assertTrue("No Flow Elements were found", flowElements > 0); + assertTrue("These Flow Elements have no check that processes them, " + + "then writes to their Element Data and expects the error. Add " + + "each to a " + CHECK + " in its module: " + missing, + missing.isEmpty()); + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FinishedElementDataTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FinishedElementDataTests.java new file mode 100644 index 000000000..d2f7c9eb5 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FinishedElementDataTests.java @@ -0,0 +1,216 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.testclasses.data.TestElementData; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +/** + * Element Data is written only by the Flow Element that provides it, or the + * Flow Elements that share it, and only while they process. Once a Flow + * Element that writes it has finished, a write through its write methods + * fails with an error, whether the Flow Element finished normally or threw. + * A Flow Element that shares the instance opens it for writing again while + * it processes. Populating it from a map is a write like any other. + * @see Specification + */ +public class FinishedElementDataTests { + + private static final Logger LOGGER = + LoggerFactory.getLogger(FinishedElementDataTests.class); + + /** + * Writes a value under its own name into the Element Data with the key + * given, which other Writers may share, and optionally throws. + */ + private static class Writer + extends FlowElementBase { + private final String key; + private final String name; + private final boolean fail; + + Writer(String key, String name, boolean fail) { + super(LOGGER, (flowData, element) -> + new TestElementData(LOGGER, flowData)); + this.key = key; + this.name = name; + this.fail = fail; + } + + @Override + protected void processInternal(FlowData data) { + data.getOrAdd(getTypedDataKey(), getDataFactory()) + .put(name, "written"); + if (fail) { + throw new IllegalStateException("thrown"); + } + } + + @Override + public String getElementDataKey() { + return key; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist( + Collections.emptyList()); + } + + @Override + public List getProperties() { + return Collections.emptyList(); + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + } + + private static void assertWriteFails(TestElementData data) { + try { + data.put("after", "value"); + fail("A write succeeded after the Flow Element had finished"); + } catch (IllegalStateException e) { + // Expected. + } + try { + data.setResult("value"); + fail("A setter succeeded after the Flow Element had finished"); + } catch (IllegalStateException e) { + // Expected. + } + try { + data.populateFromMap(Collections.singletonMap( + "after", + (Object) "value")); + fail("Populating from a map succeeded after the Flow Element " + + "had finished"); + } catch (IllegalStateException e) { + // Expected. + } + } + + private static Pipeline build(Writer... writers) throws Exception { + PipelineBuilder builder = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .setSuppressProcessException(true); + for (Writer writer : writers) { + builder.addFlowElement(writer); + } + return builder.build(); + } + + @Test + public void finished_WriteFails() throws Exception { + try (Pipeline pipeline = build(new Writer("own", "first", false)); + FlowData data = pipeline.createFlowData()) { + data.process(); + + TestElementData own = (TestElementData) data.get("own"); + assertEquals("written", own.get("first")); + assertWriteFails(own); + } + } + + @Test + public void finishedWithException_WriteFails() throws Exception { + try (Pipeline pipeline = build(new Writer("own", "first", true)); + FlowData data = pipeline.createFlowData()) { + data.process(); + + assertEquals(1, data.getErrors().size()); + assertWriteFails((TestElementData) data.get("own")); + } + } + + @Test + public void shared_EachWriterWritesWhileItProcesses() throws Exception { + try (Pipeline pipeline = build( + new Writer("shared", "first", false), + new Writer("shared", "second", false)); + FlowData data = pipeline.createFlowData()) { + data.process(); + + assertEquals(String.valueOf(data.getErrors()), + 0, data.getErrors().size()); + TestElementData shared = (TestElementData) data.get("shared"); + assertEquals("written", shared.get("first")); + assertEquals("written", shared.get("second")); + assertWriteFails(shared); + } + } + + @Test + public void beforeProcessing_WriteWorks() throws Exception { + Writer writer = new Writer("own", "first", false); + try (Pipeline pipeline = build(writer); + FlowData data = pipeline.createFlowData()) { + TestElementData own = data.getOrAdd( + writer.getTypedDataKey(), + writer.getDataFactory()); + own.put("before", "value"); + data.process(); + + assertEquals("value", own.get("before")); + assertEquals("written", own.get("first")); + } + } + + @Test + public void finished_PopulateFromMapFails() throws Exception { + try (Pipeline pipeline = build(new Writer("own", "first", false)); + FlowData data = pipeline.createFlowData()) { + data.process(); + + TestElementData own = (TestElementData) data.get("own"); + try { + own.populateFromMap(Collections.singletonMap( + "after", + (Object) "value")); + fail("Populating from a map succeeded after the Flow " + + "Element had finished"); + } catch (IllegalStateException e) { + // Expected. + } + assertFalse(own.asKeyMap().containsKey("after")); + } + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FinishedFlowDataTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FinishedFlowDataTests.java new file mode 100644 index 000000000..67c6a2078 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FinishedFlowDataTests.java @@ -0,0 +1,236 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.testclasses.data.TestElementData; +import fiftyone.pipeline.exceptions.AggregateException; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Once processing has finished, a Flow Data is read only. Adding Evidence, + * Element Data or errors fails with an error, whether processing ended + * normally or with an exception, while reads, including asking for Element + * Data that is already there, still work. Adds before and during + * processing are unaffected. + * @see Specification + */ +public class FinishedFlowDataTests { + + private static final Logger LOGGER = + LoggerFactory.getLogger(FinishedFlowDataTests.class); + + private interface Write { + void write() throws Exception; + } + + /** + * Adds Evidence, its own Element Data and an error while it processes, + * and optionally throws. + */ + private static class Writer + extends FlowElementBase { + private final boolean fail; + + Writer(boolean fail) { + super(LOGGER, (flowData, element) -> + new TestElementData(LOGGER, flowData)); + this.fail = fail; + } + + @Override + protected void processInternal(FlowData data) { + data.addEvidence("query.added", "during"); + data.getOrAdd(getTypedDataKey(), getDataFactory()); + data.addError(new RuntimeException("recorded"), this); + if (fail) { + throw new IllegalStateException("thrown"); + } + } + + @Override + public String getElementDataKey() { + return "writer"; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist( + Collections.emptyList()); + } + + @Override + public List getProperties() { + return Collections.emptyList(); + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + } + + private static void assertFails(String what, Write write) { + try { + write.write(); + fail(what + " succeeded on a Flow Data that has finished"); + } catch (Exception e) { + assertTrue( + what + " failed with " + e, + e instanceof IllegalStateException); + } + } + + private static void assertReadOnly(FlowData data, Writer writer) { + assertFails("Adding Evidence", + () -> data.addEvidence("query.after", "value")); + assertFails("Adding a map of Evidence", + () -> data.addEvidence( + Collections.singletonMap("query.after", "value"))); + assertFails("Putting Evidence into the Evidence it gives", + () -> data.getEvidence().put("query.after", "value")); + assertFails("Populating the Evidence it gives from a map", + () -> data.getEvidence().populateFromMap( + Collections.singletonMap( + "query.after", + "value"))); + assertFails("Adding Element Data by name", + () -> data.getOrAdd("other", flowData -> + new TestElementData(LOGGER, flowData))); + assertFails("Adding Element Data by typed key", + () -> data.getOrAdd( + new fiftyone.pipeline.core.typed.TypedKeyDefault<>( + "other", + TestElementData.class), + flowData -> new TestElementData(LOGGER, flowData))); + assertFails("Adding Element Data through the map", + () -> data.elementDataAsMap().put( + "other", + new TestElementData(LOGGER, data))); + assertFails("Adding an error", + () -> data.addError(new FlowError.Default( + new RuntimeException("after"), + writer))); + assertFails("Adding an exception", + () -> data.addError(new RuntimeException("after"), writer)); + + // Reads still work, including asking for Element Data that is + // already there. + assertSame( + data.get("writer"), + data.getOrAdd(writer.getTypedDataKey(), writer.getDataFactory())); + assertEquals("during", data.getEvidence().get("query.added")); + assertNull(data.getEvidence().get("query.after")); + } + + @Test + public void finished_WritesFail_ReadsWork() throws Exception { + Writer writer = new Writer(false); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .setSuppressProcessException(true) + .addFlowElement(writer) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.before", "value"); + data.process(); + + assertEquals(1, data.getErrors().size()); + assertReadOnly(data, writer); + assertEquals(1, data.getErrors().size()); + } + } + + /** + * The Element Data map taken before processing refuses writes once + * processing has finished, as one taken afterwards does. + */ + @Test + public void mapTakenBeforeProcessing_WritesFailOnceFinished() + throws Exception { + Writer writer = new Writer(false); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .setSuppressProcessException(true) + .addFlowElement(writer) + .build(); + FlowData data = pipeline.createFlowData()) { + Map map = data.elementDataAsMap(); + data.process(); + + assertFails("Putting Element Data into the map", + () -> map.put("other", new TestElementData(LOGGER, data))); + assertFails("Putting a map of Element Data into the map", + () -> map.putAll(Collections.singletonMap( + "other", + new TestElementData(LOGGER, data)))); + assertFails("Removing Element Data from the map", + () -> map.remove("writer")); + assertFails("Clearing the map", map::clear); + + // Reads through the map still work, and see what processing + // added. + assertSame(data.get("writer"), map.get("writer")); + assertFalse(map.containsKey("other")); + assertFalse(data.elementDataAsMap().containsKey("other")); + } + } + + @Test + public void finishedWithException_WritesFail() throws Exception { + Writer writer = new Writer(true); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElement(writer) + .build(); + FlowData data = pipeline.createFlowData()) { + try { + data.process(); + fail("Processing should have thrown"); + } catch (AggregateException e) { + // The Flow Element threw, and processing has still finished. + } + assertReadOnly(data, writer); + } + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataAccessorErrorTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataAccessorErrorTests.java new file mode 100644 index 000000000..0da3dcc19 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataAccessorErrorTests.java @@ -0,0 +1,140 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.ElementPropertyMetaDataDefault; +import fiftyone.pipeline.core.data.FlowData; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * FlowData's accessors throw only for something the caller did wrong. Data + * that is simply not there, because an element did not run or gave no + * value, is reported as absent rather than as an error. See + * exception-handling.md line 57: + * "The general guidance is that these functions and accessors SHOULD only + * throw errors/exceptions if the failure is likely due to something that + * the API user has done wrong." + */ +public class FlowDataAccessorErrorTests { + + private final ILoggerFactory loggerFactory = name -> mock(Logger.class); + + /** + * An element with one available property, "value". When it runs it + * adds Element Data holding the given value; when writes is false it + * adds nothing, as an element that did not run. + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + private static FlowElement element(boolean writes, Object value) + throws Exception { + FlowElement element = mock(FlowElement.class); + ElementPropertyMetaData property = new ElementPropertyMetaDataDefault( + "value", element, "test", String.class, true); + doReturn("props").when(element).getElementDataKey(); + doReturn(Collections.singletonList(property)) + .when(element).getProperties(); + when(element.hasLoadedProperties()).thenReturn(true); + doAnswer(invocation -> { + if (writes) { + FlowData flowData = invocation.getArgument(0); + ElementData data = mock(ElementData.class); + when(data.get(anyString())).thenReturn(value); + flowData.getOrAdd("props", d -> data); + } + return null; + }).when(element).process(any(FlowData.class)); + return element; + } + + private Map where(FlowElement element) + throws Exception { + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(element) + .build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.process(); + return flowData.getWhere(p -> true); + } + } + + @Test + public void getWhere_ElementThatDidNotRun_LeavesItsPropertiesOut() + throws Exception { + assertTrue(where(element(false, null)).isEmpty()); + } + + @Test + public void getWhere_PropertyWithoutAValue_LeavesItOut() + throws Exception { + assertTrue(where(element(true, null)).isEmpty()); + } + + @Test + public void getWhere_PropertyWithAValue_GivesIt() throws Exception { + assertEquals(Collections.singletonMap("props.value", "x"), + where(element(true, "x"))); + } + + @Test + public void absentData_IsNullOrNoValue_NotAnError() throws Exception { + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(element(false, null)) + .build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.number", "not a number"); + flowData.process(); + // get by key or by type throws for data that no element added, + // which FlowDataTests and the TypedKeyMap tests expect, as + // asking for it is taken as the caller's mistake. Evidence that + // is missing or of another type is not. + assertFalse(flowData.tryGetEvidence("query.number", Integer.class) + .hasValue()); + assertFalse(flowData.tryGetEvidence("query.missing", String.class) + .hasValue()); + } + } + + /** + * Keeps the List import, which the element's properties use. + */ + @SuppressWarnings("unused") + private static List unused; +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataCleanupTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataCleanupTests.java new file mode 100644 index 000000000..fc2b13670 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataCleanupTests.java @@ -0,0 +1,68 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByFiveElement; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.withSettings; + +/** + * Closing Flow Data closes the Element Data that needs cleanup and does + * nothing at all to the rest, which a cache may still hold. See + * resource-cleanup.md line 84: + * "Element Data instances that do not require cleanup MUST be left alone + * in order for result caching to work correctly." + */ +public class FlowDataCleanupTests { + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + @Test + public void close_LeavesElementDataWithoutCleanupAlone() throws Exception { + ElementData plain = mock(ElementData.class); + ElementData closeable = mock(ElementData.class, + withSettings().extraInterfaces(AutoCloseable.class)); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(new MultiplyByFiveElement(mock(Logger.class))) + .build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.getOrAdd("plain", data -> plain); + flowData.getOrAdd("closeable", data -> closeable); + } + + verifyNoInteractions(plain); + verify((AutoCloseable) closeable, times(1)).close(); + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataConcurrentWriteTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataConcurrentWriteTests.java new file mode 100644 index 000000000..8d635ce15 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataConcurrentWriteTests.java @@ -0,0 +1,236 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByFiveElement; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByTenElement; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * Flow Data of a Pipeline with parallel elements takes Element Data and + * errors from several threads, with no lock to add Element Data and errors + * kept whole. See + * thread-safety.md lines 88 to 115: + * a Flow Element adds its Element Data with one compare and swap, readers + * take no lock, and errors that members add while a group runs are added + * under a lock. + *

    + * The first writer is held inside its write until the test lets it go, + * and the second is checked from what it has done, so nothing depends on + * how long anything takes. The deadline only stops a broken run from + * hanging. + */ +public class FlowDataConcurrentWriteTests { + + private static final long DEADLINE_MILLIS = 60_000; + + /** + * Loggers that write nothing, as the errors test logs thousands. + */ + private final ILoggerFactory loggerFactory = name -> mock(Logger.class); + + private Pipeline pipeline(boolean parallel) throws Exception { + PipelineBuilder builder = new PipelineBuilder(loggerFactory); + if (parallel) { + builder.addFlowElementsParallel(new FlowElement[] { + new MultiplyByFiveElement(mock(Logger.class)), + new MultiplyByTenElement(mock(Logger.class)) }); + } else { + builder.addFlowElement( + new MultiplyByFiveElement(mock(Logger.class))); + } + return builder.build(); + } + + /** + * Two threads that add Element Data under the same key at once do not + * wait for each other, and both get the one instance that was kept. + */ + @Test + public void getOrAdd_SameKeyFromTwoThreads_NeitherWaits() + throws Exception { + try (Pipeline pipeline = pipeline(true); + FlowData flowData = pipeline.createFlowData()) { + assertTrue(pipeline.isConcurrent()); + AtomicInteger created = new AtomicInteger(); + CountDownLatch bothCreating = new CountDownLatch(2); + CountDownLatch release = new CountDownLatch(1); + AtomicReference failure = new AtomicReference<>(); + List results = new ArrayList<>(); + + FlowElement.DataFactory factory = data -> { + created.incrementAndGet(); + bothCreating.countDown(); + try { + release.await(DEADLINE_MILLIS, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return mock(ElementData.class); + }; + Runnable add = () -> { + try { + ElementData result = flowData.getOrAdd("shared", factory); + synchronized (results) { + results.add(result); + } + } catch (Throwable t) { + failure.compareAndSet(null, t); + } + }; + + Thread first = new Thread(add); + Thread second = new Thread(add); + first.start(); + second.start(); + // Both writers reach the factory while neither has finished, so + // neither waited for the other. + boolean neitherWaited = bothCreating.await( + DEADLINE_MILLIS, TimeUnit.MILLISECONDS); + release.countDown(); + first.join(DEADLINE_MILLIS); + second.join(DEADLINE_MILLIS); + + assertNull("A write failed: " + failure.get(), failure.get()); + assertTrue("One writer waited for the other.", neitherWaited); + assertEquals(2, created.get()); + assertEquals(2, results.size()); + assertSame(results.get(0), results.get(1)); + } + } + + /** + * Errors added from several threads at once are all kept. + */ + @Test + public void addError_FromSeveralThreads_KeepsEveryError() + throws Exception { + final int threads = 8; + final int perThread = 1000; + try (Pipeline pipeline = pipeline(true); + FlowData flowData = pipeline.createFlowData()) { + CountDownLatch start = new CountDownLatch(1); + FlowElement element = mock(FlowElement.class); + AtomicReference failure = new AtomicReference<>(); + List workers = new ArrayList<>(); + for (int t = 0; t < threads; t++) { + Thread worker = new Thread(() -> { + try { + start.await(); + } catch (InterruptedException e) { + return; + } + try { + for (int i = 0; i < perThread; i++) { + flowData.addError(new FlowError.Default( + new Exception("error"), element)); + } + } catch (Throwable e) { + failure.compareAndSet(null, e); + } + }); + workers.add(worker); + worker.start(); + } + start.countDown(); + for (Thread worker : workers) { + worker.join(DEADLINE_MILLIS); + assertFalse(worker.isAlive()); + } + assertNull("A write failed: " + failure.get(), failure.get()); + assertEquals(threads * perThread, flowData.getErrors().size()); + } + } + + /** + * Without parallel elements, adding Element Data takes no lock, so a + * second writer for another key is not held up by the first. + */ + @Test + public void getOrAdd_WithoutParallelElements_TakesNoLock() + throws Exception { + try (Pipeline pipeline = pipeline(false); + FlowData flowData = pipeline.createFlowData()) { + assertFalse(pipeline.isConcurrent()); + CountDownLatch firstCreating = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicReference failure = new AtomicReference<>(); + + Thread first = new Thread(() -> { + try { + flowData.getOrAdd("first", data -> { + firstCreating.countDown(); + try { + release.await( + DEADLINE_MILLIS, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return mock(ElementData.class); + }); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } + }); + first.start(); + assertTrue(firstCreating.await( + DEADLINE_MILLIS, TimeUnit.MILLISECONDS)); + Thread second = new Thread(() -> { + try { + flowData.getOrAdd("second", + data -> mock(ElementData.class)); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } + }); + second.start(); + second.join(DEADLINE_MILLIS); + boolean secondFinished = second.isAlive() == false; + release.countDown(); + first.join(DEADLINE_MILLIS); + + assertNull("A write failed: " + failure.get(), failure.get()); + assertTrue("The second writer waited for the first.", + secondFinished); + } + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataDeclaredExceptionsTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataDeclaredExceptionsTests.java new file mode 100644 index 000000000..3f165ac37 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataDeclaredExceptionsTests.java @@ -0,0 +1,164 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.core.exceptions.PipelineDataException; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByFiveElement; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + +/** + * The FlowData interface declares, in the throws clause that IDEs and + * reflection read, the exceptions its methods throw for a mistake by the + * caller, and the implementation throws one of those. See + * exception-handling.md line 17: + * "In all cases, standard language features SHOULD be used to let IDEs and + * users know what exceptions/errors a particular function can throw." + */ +public class FlowDataDeclaredExceptionsTests { + + private final ILoggerFactory loggerFactory = name -> mock(Logger.class); + + private interface Call { + void run(FlowData flowData) throws Exception; + } + + private static List> declared(String name, Class... types) + throws Exception { + Method method = FlowData.class.getMethod(name, types); + return Arrays.asList(method.getExceptionTypes()); + } + + private static void assertDeclares( + List> declared, Class type, String method) { + assertTrue(method + " does not declare " + type.getSimpleName() + + ", only " + declared + ".", + declared.contains(type)); + } + + /** + * Call the method on Flow Data that has not been processed, and check + * the exception is one the interface declares. + */ + private void assertThrowsDeclared( + List> declared, String method, boolean process, Call call) + throws Exception { + MultiplyByFiveElement element = + new MultiplyByFiveElement(mock(Logger.class)); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(element) + .build(); + FlowData flowData = pipeline.createFlowData()) { + if (process) { + flowData.addEvidence(element.evidenceKeys.get(0), 2); + flowData.process(); + } + try { + call.run(flowData); + fail(method + " did not throw."); + } catch (Exception e) { + boolean isDeclared = false; + for (Class type : declared) { + isDeclared |= type.isInstance(e); + } + assertTrue(method + " threw " + e.getClass().getName() + + ", which it does not declare, only " + declared + ".", + isDeclared); + } + } + } + + @Test + public void process_DeclaresAndThrowsIllegalState() throws Exception { + List> declared = declared("process"); + assertDeclares(declared, IllegalStateException.class, "process"); + assertThrowsDeclared(declared, "process", true, FlowData::process); + } + + @Test + @SuppressWarnings("unchecked") + public void getByTypedKey_DeclaresAndThrowsIllegalState() + throws Exception { + List> declared = declared("get", TypedKey.class); + assertDeclares(declared, IllegalStateException.class, "get(TypedKey)"); + assertDeclares(declared, IllegalArgumentException.class, + "get(TypedKey)"); + assertThrowsDeclared(declared, "get(TypedKey)", false, + d -> d.get(mock(TypedKey.class))); + } + + @Test + public void getByName_DeclaresAndThrowsIllegalState() throws Exception { + List> declared = declared("get", String.class); + assertDeclares(declared, IllegalStateException.class, "get(String)"); + assertThrowsDeclared(declared, "get(String)", false, + d -> d.get("muliplyByFive")); + } + + @Test + public void getFromElement_DeclaresAndThrowsIllegalState() + throws Exception { + List> declared = + declared("getFromElement", FlowElement.class); + assertDeclares(declared, IllegalStateException.class, + "getFromElement"); + assertDeclares(declared, IllegalArgumentException.class, + "getFromElement"); + assertThrowsDeclared(declared, "getFromElement", true, + d -> d.getFromElement(null)); + } + + @Test + public void getAs_DeclaresAndThrowsPipelineDataException() + throws Exception { + List> declared = declared("getAs", String.class, Class.class); + assertDeclares(declared, PipelineDataException.class, "getAs"); + assertDeclares(declared, ClassCastException.class, "getAs"); + assertThrowsDeclared(declared, "getAs", false, + d -> d.getAs("result", Integer.class)); + for (String name : new String[] { + "getAsString", "getAsBool", "getAsInt", "getAsLong", + "getAsFloat", "getAsDouble" }) { + assertDeclares(declared(name, String.class), + PipelineDataException.class, name); + } + } + + /** + * Keeps the ElementData import, which documents what get returns. + */ + @SuppressWarnings("unused") + private static ElementData unused; +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataTests.java index 613033fa2..70a741e2f 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataTests.java @@ -123,11 +123,13 @@ public void FlowData_AddEvidenceMap() { @Test public void FlowData_AddDataString_GetDataString() { - flowData.process(); String key = "key"; ElementData data = flowData.getOrAdd( key, (FlowElement.DataFactory) flowData -> new TestElementData(mock(Logger.class), flowData)); + // Element Data is added before processing finishes, as a Flow Data + // is read only afterwards. + flowData.process(); Object result = flowData.get(key); @@ -135,13 +137,44 @@ public void FlowData_AddDataString_GetDataString() { } @Test - public void FlowData_AddDataString_GetDataTypedKey() { + public void FlowData_AddDataString_GetDataStringOtherCase() { + ElementData data = flowData.getOrAdd( + "device", + (FlowElement.DataFactory) flowData -> new TestElementData(mock(Logger.class), flowData)); + // Element Data is added before processing finishes, as a Flow Data + // is read only afterwards. + flowData.process(); + + assertEquals(data, flowData.get("Device")); + assertEquals(data, flowData.get("DEVICE")); + } + + @Test + public void FlowData_GetOrAddOtherCase_SameData() { + ElementData data = flowData.getOrAdd( + "device", + (FlowElement.DataFactory) flowData -> new TestElementData(mock(Logger.class), flowData)); + // Element Data is added before processing finishes, as a Flow Data + // is read only afterwards. flowData.process(); + + ElementData again = flowData.getOrAdd( + "Device", + (FlowElement.DataFactory) flowData -> new TestElementData(mock(Logger.class), flowData)); + + assertSame(data, again); + } + + @Test + public void FlowData_AddDataString_GetDataTypedKey() { String key = "key"; TypedKey typedKey = new TypedKeyDefault<>(key); ElementData data = flowData.getOrAdd( key, (FlowElement.DataFactory) flowData -> new TestElementData(mock(Logger.class), flowData)); + // Element Data is added before processing finishes, as a Flow Data + // is read only afterwards. + flowData.process(); Object result = flowData.get(typedKey); @@ -150,8 +183,6 @@ public void FlowData_AddDataString_GetDataTypedKey() { @Test public void FlowData_AddDataString_GetDataElement() { - flowData.process(); - TestElement element = new TestElement( mock(Logger.class), (flowData, flowElement) -> new TestElementData( @@ -160,6 +191,9 @@ public void FlowData_AddDataString_GetDataElement() { ElementData data = flowData.getOrAdd( element.getElementDataKey(), element.getDataFactory()); + // Element Data is added before processing finishes, as a Flow Data + // is read only afterwards. + flowData.process(); Object result = flowData.getFromElement(element); @@ -168,12 +202,14 @@ public void FlowData_AddDataString_GetDataElement() { @Test public void FlowData_AddDataTypedKey_GetDataString() { - flowData.process(); String key = "key"; TypedKey typedKey = new TypedKeyDefault<>(key); TestElementData data = flowData.getOrAdd( typedKey, flowData -> new TestElementData(mock(Logger.class), flowData)); + // Element Data is added before processing finishes, as a Flow Data + // is read only afterwards. + flowData.process(); Object result = flowData.get(key); @@ -182,12 +218,14 @@ public void FlowData_AddDataTypedKey_GetDataString() { @Test public void FlowData_AddDataTypedKey_GetDataTypedKey() { - flowData.process(); String key = "key"; TypedKey typedKey = new TypedKeyDefault<>(key); TestElementData data = flowData.getOrAdd( typedKey, flowData -> new TestElementData(mock(Logger.class), flowData)); + // Element Data is added before processing finishes, as a Flow Data + // is read only afterwards. + flowData.process(); Object result = flowData.get(typedKey); @@ -196,7 +234,6 @@ public void FlowData_AddDataTypedKey_GetDataTypedKey() { @Test public void FlowData_AddDataTypedKey_GetDataElement() { - flowData.process(); TestElement element = new TestElement( mock(Logger.class), (flowData, flowElement) -> new TestElementData( @@ -207,6 +244,9 @@ public void FlowData_AddDataTypedKey_GetDataElement() { TestElementData data = flowData.getOrAdd( typedKey, element.getDataFactory()); + // Element Data is added before processing finishes, as a Flow Data + // is read only afterwards. + flowData.process(); Object result = flowData.getFromElement(element); @@ -229,11 +269,13 @@ public void FlowData_GetDataAsMap() { @Test public void FlowData_GetDataAsEnumerable() { - flowData.process(); String key = "key"; TestElementData data = flowData.getOrAdd( key, flowData -> new TestElementData(mock(Logger.class), flowData)); + // Element Data is added before processing finishes, as a Flow Data + // is read only afterwards. + flowData.process(); Iterable result = flowData.elementDataAsIterable(); @@ -247,12 +289,14 @@ public void FlowData_GetDataAsEnumerable() { @Test public void FlowData_UpdateDataAsMap() { - flowData.process(); String key = "key"; TestElementData data = new TestElementData(mock(Logger.class), flowData); Map dataAsMap = flowData.elementDataAsMap(); dataAsMap.put(key, data); + // Element Data is added before processing finishes, as a Flow Data + // is read only afterwards. + flowData.process(); Object result = flowData.get(key); assertEquals(data, result); @@ -317,6 +361,25 @@ public void FlowData_GetNotPresent_Element() { assertNull(result); } + /** + * The error logged for a processing failure carries the exception, so + * the log says why processing failed. + */ + @Test + public void FlowData_AddError_LogsTheException() throws Exception { + Logger logger = mock(Logger.class); + when(logger.isErrorEnabled()).thenReturn(true); + Exception cause = new Exception("The cause of the failure."); + try (FlowData data = new FlowDataDefault( + logger, + pipeline, + new EvidenceDefault(mock(Logger.class)))) { + + data.addError(cause, mock(FlowElement.class)); + } + verify(logger).error(anyString(), same(cause)); + } + @Test(expected = IllegalStateException.class) public void FlowData_PipelineDisposed() throws Exception { when(pipeline.isClosed()).thenReturn(true); @@ -653,13 +716,14 @@ private interface CloseableData extends ElementData, AutoCloseable { */ @Test public void FlowData_Close() throws Exception { - flowData.process(); - CloseableData data = mock(CloseableData.class); flowData.getOrAdd( "test", (FlowElement.DataFactory) flowData -> data); + // Element Data is added before processing finishes, as a Flow Data + // is read only afterwards. + flowData.process(); CloseFlowData(); verify(data, times(1)).close(); } @@ -671,11 +735,12 @@ public void FlowData_Close() throws Exception { */ @Test public void FlowData_CloseNotCloseable() throws Exception { - flowData.process(); - ElementData data = mock(ElementData.class); flowData.getOrAdd("test", flowData -> data); + // Element Data is added before processing finishes, as a Flow Data + // is read only afterwards. + flowData.process(); CloseFlowData(); } } diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataThrowsTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataThrowsTests.java new file mode 100644 index 000000000..316d2a5a4 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowDataThrowsTests.java @@ -0,0 +1,187 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineDataException; +import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.core.typed.TypedKeyDefault; +import fiftyone.pipeline.exceptions.AggregateException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.fail; +import static org.mockito.Mockito.*; + +/** + * Each exception a FlowData method throws is declared by that method of the + * FlowData interface, by its own type or a type it extends that is more + * specific than RuntimeException, so that IDEs and callers can see it. + */ +public class FlowDataThrowsTests { + + private PipelineInternal pipeline; + + private FlowDataDefault flowData; + + /** + * Code that is expected to throw. + */ + private interface Call { + void call() throws Exception; + } + + @Before + public void init() { + pipeline = mock(PipelineInternal.class); + flowData = new FlowDataDefault( + mock(Logger.class), + pipeline, + new EvidenceDefault(mock(Logger.class))); + } + + @After + public void cleanUp() throws Exception { + flowData.close(); + } + + @Test + public void process_Twice() throws Exception { + flowData.process(); + assertDeclared(method("process"), () -> flowData.process()); + } + + @Test + public void process_PipelineClosed() throws Exception { + when(pipeline.isClosed()).thenReturn(true); + assertDeclared(method("process"), () -> flowData.process()); + } + + @Test + public void process_ElementsFailed() throws Exception { + doThrow(new AggregateException( + "Exception(s) occurred processing evidence.", + Collections.singletonList(new Exception("failed")))) + .when(pipeline).process(flowData); + assertDeclared(method("process"), () -> flowData.process()); + } + + @Test + public void getByTypedKey_NotProcessed() throws Exception { + TypedKey key = + new TypedKeyDefault<>("test", ElementData.class); + assertDeclared( + method("get", TypedKey.class), + () -> flowData.get(key)); + } + + @Test + public void getByDataKey_NotProcessed() throws Exception { + assertDeclared( + method("get", String.class), + () -> flowData.get("test")); + } + + @Test + @SuppressWarnings("unchecked") + public void getFromElement_NotProcessed() throws Exception { + FlowElement element = mock(FlowElement.class); + when(element.getTypedDataKey()) + .thenReturn(new TypedKeyDefault<>("test", ElementData.class)); + assertDeclared( + method("getFromElement", FlowElement.class), + () -> flowData.getFromElement(element)); + } + + @Test + public void getAs_NotProcessed() throws Exception { + assertDeclared( + method("getAs", String.class, Class.class), + () -> flowData.getAs("test", String.class)); + } + + @Test + public void getAs_PropertyNotKnown() throws Exception { + flowData.process(); + when(pipeline.getMetaDataForProperty("test")) + .thenThrow(new PipelineDataException("Property not found")); + assertDeclared( + method("getAs", String.class, Class.class), + () -> flowData.getAs("test", String.class)); + } + + @Test + public void typedGetters_NotProcessed() throws Exception { + assertDeclared(method("getAsString", String.class), + () -> flowData.getAsString("test")); + assertDeclared(method("getAsBool", String.class), + () -> flowData.getAsBool("test")); + assertDeclared(method("getAsInt", String.class), + () -> flowData.getAsInt("test")); + assertDeclared(method("getAsLong", String.class), + () -> flowData.getAsLong("test")); + assertDeclared(method("getAsFloat", String.class), + () -> flowData.getAsFloat("test")); + assertDeclared(method("getAsDouble", String.class), + () -> flowData.getAsDouble("test")); + } + + private static Method method(String name, Class... parameters) + throws NoSuchMethodException { + return FlowData.class.getMethod(name, parameters); + } + + /** + * Run the call, which must throw, and check that the interface method + * declares what it threw. + */ + private static void assertDeclared(Method method, Call call) { + Throwable thrown = null; + try { + call.call(); + } catch (Throwable e) { + thrown = e; + } + if (thrown == null) { + fail(method.getName() + " should have thrown"); + } + for (Class declared : method.getExceptionTypes()) { + if (declared != RuntimeException.class && + declared != Exception.class && + declared != Throwable.class && + declared.isInstance(thrown)) { + return; + } + } + fail("FlowData." + method.getName() + " throws " + + thrown.getClass().getName() + " but declares " + + Arrays.toString(method.getExceptionTypes())); + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowElementContractTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowElementContractTests.java new file mode 100644 index 000000000..dafb7e33f --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/FlowElementContractTests.java @@ -0,0 +1,73 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Every Flow Element has to give the Evidence keys it uses, the Properties + * it populates and its Data Key, because the FlowElement interface declares + * all three without a default, so no element compiles without them. See + * conceptual-overview.md line 73: + * "Each Element MUST:" advertize the Evidence keys that it can make use of, + * publicize details of the Properties it will populate, and expose the + * string Data Key that is used to identify its Element Data within Flow + * Data. + */ +public class FlowElementContractTests { + + private static void assertRequired( + String name, Class returnType) throws Exception { + Method method = FlowElement.class.getMethod(name); + assertTrue(name + " is not public.", + Modifier.isPublic(method.getModifiers())); + assertTrue(name + " is not abstract, so an element need not give it.", + Modifier.isAbstract(method.getModifiers())); + assertFalse(name + " has a default.", method.isDefault()); + assertEquals(returnType, method.getReturnType()); + } + + @Test + public void evidenceKeys_AreRequired() throws Exception { + assertRequired("getEvidenceKeyFilter", EvidenceKeyFilter.class); + } + + @Test + public void properties_AreRequired() throws Exception { + assertRequired("getProperties", List.class); + } + + @Test + public void dataKey_IsRequired() throws Exception { + assertRequired("getElementDataKey", String.class); + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelElementDataAddTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelElementDataAddTests.java new file mode 100644 index 000000000..19e1e7118 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelElementDataAddTests.java @@ -0,0 +1,157 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.ElementDataBase; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Members of a parallel group add their Element Data to the Flow Data + * without a lock, so one member adding its own never waits for another + * member that is still making its instance. + * @see Specification + */ +public class ParallelElementDataAddTests { + + private static final Logger LOGGER = + LoggerFactory.getLogger(ParallelElementDataAddTests.class); + + private static final long WAIT_SECONDS = 5; + + public static class MemberData extends ElementDataBase { + public MemberData(Logger logger, FlowData flowData) { + super(logger, flowData); + } + } + + /** + * Adds its own Element Data. The slow member's data factory waits until + * the quick member has added its own, as making an instance can take a + * while. + */ + private static class Member + extends FlowElementBase { + private final String key; + + Member(String key, CountDownLatch beforeAdd, CountDownLatch waitFor, + CountDownLatch added, AtomicBoolean quickAdded) { + super(LOGGER, (flowData, element) -> { + if (beforeAdd != null) { + beforeAdd.countDown(); + } + if (waitFor != null) { + try { + quickAdded.set( + waitFor.await(WAIT_SECONDS, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + return new MemberData(LOGGER, flowData); + }); + this.key = key; + this.beforeAdd = waitFor == null ? beforeAdd : null; + this.added = added; + } + + private final CountDownLatch beforeAdd; + private final CountDownLatch added; + + @Override + protected void processInternal(FlowData data) throws Exception { + if (beforeAdd != null) { + // The quick member starts once the slow one is making its + // instance. + assertTrue(beforeAdd.await(WAIT_SECONDS, TimeUnit.SECONDS)); + } + data.getOrAdd(getTypedDataKey(), getDataFactory()); + if (added != null) { + added.countDown(); + } + } + + @Override + public String getElementDataKey() { + return key; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist(Collections.emptyList()); + } + + @Override + public List getProperties() { + return Collections.emptyList(); + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + } + + @Test + @SuppressWarnings("rawtypes") + public void parallelMembers_AddWithoutWaitingForEachOther() + throws Exception { + CountDownLatch slowMaking = new CountDownLatch(1); + CountDownLatch quickAddedLatch = new CountDownLatch(1); + AtomicBoolean quickAdded = new AtomicBoolean(false); + Member slow = new Member( + "slow", slowMaking, quickAddedLatch, null, quickAdded); + Member quick = new Member( + "quick", slowMaking, null, quickAddedLatch, quickAdded); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElementsParallel(new FlowElement[]{slow, quick}) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + data.process(); + assertTrue( + "The quick member's add waited for the slow member's", + quickAdded.get()); + assertNotNull(data.get("slow")); + assertNotNull(data.get("quick")); + } + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelElementsTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelElementsTests.java index 617089b24..862bb70ba 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelElementsTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelElementsTests.java @@ -130,10 +130,11 @@ public TestElementData create(FlowData flowData) { mock(Logger.class), pipeline, new EvidenceDefault(mock(Logger.class)))) { - data.process(); - // Act parallelElements.process(data); + // The members add their Element Data before processing + // finishes, as a Flow Data is read only afterwards. + data.process(); List startTimes = new ArrayList<>(); startTimes.add((Date) data.get("element1").get("start")); diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelErrorsLockTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelErrorsLockTests.java new file mode 100644 index 000000000..04826b6b6 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/ParallelErrorsLockTests.java @@ -0,0 +1,237 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Members of a parallel group add the errors they record under a lock while + * the group runs, so they appear in the order the members add them. Every + * other add, and every read once processing has finished, takes no lock. + * The lock is the Flow Data's errors collection, which these tests hold to + * see whether an add waits for it. + * @see Specification + */ +public class ParallelErrorsLockTests { + + private static final Logger LOGGER = + LoggerFactory.getLogger(ParallelErrorsLockTests.class); + + private static final long WAIT_SECONDS = 5; + + /** + * How long an add that waits for the lock is given to finish anyway. + */ + private static final long HELD_MILLISECONDS = 300; + + /** + * Adds one error of its own while it processes, saying just before and + * just after. Members given a barrier wait for each other first, so + * they add at the same time. + */ + private static class ErrorMember + extends FlowElementBase { + private final String key; + private final CountDownLatch adding = new CountDownLatch(1); + private final CountDownLatch added = new CountDownLatch(1); + private final CyclicBarrier together; + + ErrorMember(String key, CyclicBarrier together) { + super(LOGGER, (flowData, element) -> null); + this.key = key; + this.together = together; + } + + @Override + protected void processInternal(FlowData data) throws Exception { + if (together != null) { + together.await(WAIT_SECONDS, TimeUnit.SECONDS); + } + adding.countDown(); + data.addError(new Exception(key), this); + added.countDown(); + } + + @Override + public String getElementDataKey() { + return key; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist(Collections.emptyList()); + } + + @Override + public List getProperties() { + return Collections.emptyList(); + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + } + + private static Object errorsLock(FlowData data) throws Exception { + Field field = FlowDataDefault.class.getDeclaredField("errors"); + field.setAccessible(true); + return field.get(data); + } + + private static List messages(Collection errors) { + List messages = new ArrayList<>(); + for (FlowError error : errors) { + messages.add(error.getThrowable().getMessage()); + } + Collections.sort(messages); + return messages; + } + + @Test + @SuppressWarnings("rawtypes") + public void memberError_AddedUnderTheLockWhileTheGroupRuns() + throws Exception { + ErrorMember member = new ErrorMember("member", null); + ErrorMember other = new ErrorMember("other", null); + ExecutorService caller = Executors.newSingleThreadExecutor(); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .setSuppressProcessException(true) + .addFlowElementsParallel(new FlowElement[]{member, other}) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + Future processed; + synchronized (errorsLock(data)) { + processed = caller.submit(() -> { + data.process(); + return null; + }); + assertTrue( + member.adding.await(WAIT_SECONDS, TimeUnit.SECONDS)); + assertFalse( + "A member added its error without the errors lock", + member.added.await( + HELD_MILLISECONDS, + TimeUnit.MILLISECONDS)); + } + processed.get(WAIT_SECONDS, TimeUnit.SECONDS); + assertTrue(member.added.await(WAIT_SECONDS, TimeUnit.SECONDS)); + assertEquals( + Arrays.asList("member", "other"), + messages(data.getErrors())); + } finally { + caller.shutdownNow(); + } + } + + @Test + @SuppressWarnings("rawtypes") + public void errorOutsideTheGroup_AddedWithNoLock() throws Exception { + ErrorMember before = new ErrorMember("before", null); + ErrorMember first = new ErrorMember("first", null); + ErrorMember second = new ErrorMember("second", null); + ExecutorService caller = Executors.newSingleThreadExecutor(); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .setSuppressProcessException(true) + .addFlowElement(before) + .addFlowElementsParallel(new FlowElement[]{first, second}) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + Object lock = errorsLock(data); + Future processed; + synchronized (lock) { + processed = caller.submit(() -> { + data.process(); + return null; + }); + assertTrue( + "An error added outside the group waited for the lock", + before.added.await(WAIT_SECONDS, TimeUnit.SECONDS)); + } + processed.get(WAIT_SECONDS, TimeUnit.SECONDS); + synchronized (lock) { + Future> read = + caller.submit(data::getErrors); + assertEquals( + "Reading the errors once processing has finished " + + "waited for the lock", + 3, + read.get(WAIT_SECONDS, TimeUnit.SECONDS).size()); + } + } finally { + caller.shutdownNow(); + } + } + + @Test + @SuppressWarnings("rawtypes") + public void membersAddErrorsAtOnce_AllPresent() throws Exception { + CyclicBarrier together = new CyclicBarrier(2); + ErrorMember first = new ErrorMember("first", together); + ErrorMember second = new ErrorMember("second", together); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .setSuppressProcessException(true) + .addFlowElementsParallel(new FlowElement[]{first, second}) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + data.process(); + assertEquals( + Arrays.asList("first", "second"), + messages(data.getErrors())); + } + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineAutoCloseTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineAutoCloseTests.java new file mode 100644 index 000000000..737a2a5da --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineAutoCloseTests.java @@ -0,0 +1,88 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByFiveElement; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * Closing a Pipeline closes its elements only when it is set to, so that + * an element shared with another Pipeline can be kept open. See + * resource-cleanup.md line 28: + * "However, if the multi Pipeline Elements feature is implemented then + * there MUST be an option to disable this behavior." + */ +public class PipelineAutoCloseTests { + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + @Test + public void autoCloseOff_SharedElementStaysOpen() throws Exception { + MultiplyByFiveElement shared = + new MultiplyByFiveElement(mock(Logger.class)); + try (Pipeline first = new PipelineBuilder(loggerFactory) + .setAutoCloseElements(false) + .addFlowElement(shared) + .build(); + Pipeline second = new PipelineBuilder(loggerFactory) + .setAutoCloseElements(false) + .addFlowElement(shared) + .build()) { + // Closing again at the end of the block does nothing. + first.close(); + + assertFalse("Closing one Pipeline closed the shared element.", + shared.isClosed()); + try (FlowData flowData = second.createFlowData()) { + flowData.addEvidence(shared.evidenceKeys.get(0), 2); + flowData.process(); + assertEquals(10, flowData.getFromElement(shared).getResult()); + } + } finally { + shared.close(); + } + } + + @Test + public void autoCloseOn_ClosesTheElement() throws Exception { + MultiplyByFiveElement element = + new MultiplyByFiveElement(mock(Logger.class)); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setAutoCloseElements(true) + .addFlowElement(element) + .build()) { + assertFalse(element.isClosed()); + } + assertTrue(element.isClosed()); + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBaseTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBaseTests.java index cc5b74fd8..22ad433b5 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBaseTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineBuilderBaseTests.java @@ -22,12 +22,15 @@ package fiftyone.pipeline.core.flowelements; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import org.junit.Test; +import fiftyone.pipeline.core.Constants; import fiftyone.pipeline.core.testclasses.services.TestAutoCloseableService; import fiftyone.pipeline.core.testclasses.services.TestCloseableService; @@ -53,4 +56,21 @@ public void PipelineBuilderBase_Build() throws Exception { verify(service2, times(1)).close(); } + /** + * The builder's defaults come from {@link Constants} and keep the values + * they have always had. + */ + @Test + public void PipelineBuilderBase_Defaults() { + TestPipelineBuilder builder = new TestPipelineBuilder(); + assertEquals( + Constants.DEFAULT_AUTO_CLOSE_ELEMENTS, + builder.autoCloseElements); + assertEquals( + Constants.DEFAULT_SUPPRESS_PROCESS_EXCEPTIONS, + builder.suppressProcessExceptions); + assertFalse(Constants.DEFAULT_AUTO_CLOSE_ELEMENTS); + assertFalse(Constants.DEFAULT_SUPPRESS_PROCESS_EXCEPTIONS); + } + } diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineEvidenceKeyFilterTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineEvidenceKeyFilterTests.java new file mode 100644 index 000000000..7bb883866 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineEvidenceKeyFilterTests.java @@ -0,0 +1,105 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByFiveElement; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * The Pipeline's Evidence key filter depends only on the Flow Elements it + * holds, which never change after it is built. So the Pipeline builds the + * filter once, when it is built, and every request then reads it with no + * lock, as usage sharing and the web integration do. A final field set in + * the constructor is seen whole by every thread. + * @see Specification + */ +public class PipelineEvidenceKeyFilterTests { + + private static final int THREADS = 8; + + @Test + public void evidenceKeyFilter_HeldInAFinalField() throws Exception { + Field field = + PipelineDefault.class.getDeclaredField("evidenceKeyFilter"); + assertTrue( + "The filter is built with the Pipeline, so its field is final", + Modifier.isFinal(field.getModifiers())); + } + + @Test + public void evidenceKeyFilter_BuiltWithThePipeline() throws Exception { + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElement(new MultiplyByFiveElement( + LoggerFactory.getLogger(MultiplyByFiveElement.class))) + .build()) { + Field field = + PipelineDefault.class.getDeclaredField("evidenceKeyFilter"); + field.setAccessible(true); + // Read the field before anything asks for the filter. + Object built = field.get(pipeline); + assertNotNull( + "The filter exists as soon as the Pipeline is built", + built); + assertTrue( + "The filter accepts the Flow Element's Evidence", + ((EvidenceKeyFilter) built).include("value")); + assertSame(built, pipeline.getEvidenceKeyFilter()); + } + } + + @Test + public void evidenceKeyFilter_SameFilterForEveryThread() throws Exception { + ExecutorService threads = Executors.newFixedThreadPool(THREADS); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .build()) { + List> calls = new ArrayList<>(); + for (int i = 0; i < THREADS; i++) { + calls.add(pipeline::getEvidenceKeyFilter); + } + List> filters = threads.invokeAll(calls); + EvidenceKeyFilter first = filters.get(0).get(); + for (Future filter : filters) { + assertSame(first, filter.get()); + } + } finally { + threads.shutdownNow(); + } + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineImmutableTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineImmutableTests.java new file mode 100644 index 000000000..a63e24b46 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineImmutableTests.java @@ -0,0 +1,109 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByFiveElement; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByTenElement; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + +/** + * Once built, a Pipeline's elements cannot be changed. See + * conceptual-overview.md line 118: + * "A Pipeline instance MUST be immutable. In other words, once created, it + * cannot be changed by adding new elements, removing old ones, etc." + */ +public class PipelineImmutableTests { + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + /** + * The list of elements a Pipeline gives cannot be changed. + */ + @Test + public void getFlowElements_CannotBeChanged() throws Exception { + MultiplyByFiveElement five = new MultiplyByFiveElement( + mock(Logger.class)); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(five) + .build()) { + try { + pipeline.getFlowElements().add( + new MultiplyByTenElement(mock(Logger.class))); + fail("An element was added to a built Pipeline."); + } catch (UnsupportedOperationException e) { + // Expected. + } + try { + pipeline.getFlowElements().clear(); + fail("The elements of a built Pipeline were removed."); + } catch (UnsupportedOperationException e) { + // Expected. + } + assertEquals(Collections.singletonList(five), + pipeline.getFlowElements()); + } + } + + /** + * An element added to the builder after it has built a Pipeline is not + * part of that Pipeline, and does not run when it processes. + */ + @Test + public void builderReused_DoesNotChangeABuiltPipeline() throws Exception { + MultiplyByFiveElement five = new MultiplyByFiveElement( + mock(Logger.class)); + MultiplyByTenElement ten = new MultiplyByTenElement( + mock(Logger.class)); + PipelineBuilder builder = new PipelineBuilder(loggerFactory) + .addFlowElement(five); + try (Pipeline pipeline = builder.build()) { + builder.addFlowElement(ten); + + assertEquals(1, pipeline.getFlowElements().size()); + assertSame(five, pipeline.getFlowElements().get(0)); + assertEquals(null, pipeline.getElement(MultiplyByTenElement.class)); + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence(five.evidenceKeys.get(0), 2); + flowData.process(); + assertEquals(10, flowData.getFromElement(five).getResult()); + assertFalse( + "An element added to the builder later ran.", + flowData.getDataKeys().stream().anyMatch( + k -> k.equalsIgnoreCase(ten.getElementDataKey()))); + } + } + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineSuppressProcessExceptionsTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineSuppressProcessExceptionsTests.java new file mode 100644 index 000000000..756802f18 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/PipelineSuppressProcessExceptionsTests.java @@ -0,0 +1,77 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * The suppress process exceptions setting can be read through the public + * Pipeline interface, so that a wrapper such as the web integration can + * decide from it. See + * exception-handling.md: + * "This property MUST be exposed as a public property as part of the + * `Pipeline` public interface as `IsSuppressProcessExceptions`". + */ +public class PipelineSuppressProcessExceptionsTests { + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + /** + * The Pipeline interface declares a public isSuppressProcessExceptions + * that returns a boolean. + */ + @Test + public void pipelineInterface_DeclaresIsSuppressProcessExceptions() + throws Exception { + Method method = Pipeline.class.getMethod("isSuppressProcessExceptions"); + assertEquals(boolean.class, method.getReturnType()); + assertTrue(Modifier.isPublic(method.getModifiers())); + } + + /** + * The value read through the interface is the one the builder set, for + * both settings. + */ + @Test + public void isSuppressProcessExceptions_GivesTheBuilderSetting() + throws Exception { + for (boolean suppress : new boolean[] { true, false }) { + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(suppress) + .build()) { + Method method = + Pipeline.class.getMethod("isSuppressProcessExceptions"); + assertEquals(suppress, method.invoke(pipeline)); + } + } + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/RunsLastTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/RunsLastTests.java new file mode 100644 index 000000000..65dfec4dc --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/RunsLastTests.java @@ -0,0 +1,145 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.configuration.ElementOptions; +import fiftyone.pipeline.core.configuration.PipelineOptions; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByElement; +import fiftyone.pipeline.core.testclasses.flowelements.RunsLastElement; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * A Flow Element that must run last, as the usage sharing element must, is + * refused anywhere but last, and in a parallel group, whether the Pipeline + * is built in code or from a configuration. + * @see Specification + */ +@SuppressWarnings("rawtypes") +public class RunsLastTests { + + private static final String MUST_BE_LAST = + "must be the last Flow Element and outside any parallel group"; + + private static void assertRefused(PipelineBuilder builder) { + try (Pipeline pipeline = builder.build()) { + fail("A Pipeline with the element anywhere but last was built."); + } catch (PipelineConfigurationException e) { + assertTrue(e.getMessage(), e.getMessage().contains(MUST_BE_LAST)); + assertTrue(e.getMessage(), + e.getMessage().startsWith("The test element")); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + @Test + public void RunsLast_First_Refused() { + assertRefused(new PipelineBuilder() + .addFlowElement(new RunsLastElement()) + .addFlowElement(new MultiplyByElement(2))); + } + + @Test + public void RunsLast_InAParallelGroup_Refused() { + assertRefused(new PipelineBuilder() + .addFlowElement(new MultiplyByElement(2)) + .addFlowElementsParallel(new FlowElement[]{ + new MultiplyByElement(3), + new RunsLastElement()})); + } + + @Test + public void RunsLast_Last_Built() throws Exception { + RunsLastElement last = new RunsLastElement(); + try (Pipeline pipeline = new PipelineBuilder() + .addFlowElement(new MultiplyByElement(2)) + .addFlowElement(last) + .build()) { + List elements = pipeline.getFlowElements(); + assertEquals(last, elements.get(elements.size() - 1)); + } + } + + private static ElementOptions element(String builderName) { + ElementOptions element = new ElementOptions(); + element.builderName = builderName; + return element; + } + + private static ElementOptions multiplyBy() { + ElementOptions element = element("MultiplyByElement"); + element.buildParameters.put("Multiple", "8"); + return element; + } + + private static void assertRefusedFrom(PipelineOptions options) { + try (Pipeline pipeline = + new PipelineBuilder().buildFromConfiguration(options)) { + fail("A Pipeline with the element anywhere but last was built " + + "from a configuration."); + } catch (PipelineConfigurationException e) { + assertTrue(e.getMessage(), e.getMessage().contains(MUST_BE_LAST)); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + @Test + public void RunsLast_FirstInAConfiguration_Refused() { + PipelineOptions options = new PipelineOptions(); + options.elements.add(element("RunsLastElement")); + options.elements.add(multiplyBy()); + assertRefusedFrom(options); + } + + @Test + public void RunsLast_InAParallelGroupInAConfiguration_Refused() { + ElementOptions group = new ElementOptions(); + group.subElements.add(multiplyBy()); + group.subElements.add(element("RunsLastElement")); + PipelineOptions options = new PipelineOptions(); + options.elements.add(multiplyBy()); + options.elements.add(group); + assertRefusedFrom(options); + } + + @Test + public void RunsLast_LastInAConfiguration_Built() throws Exception { + PipelineOptions options = new PipelineOptions(); + options.elements.add(multiplyBy()); + options.elements.add(element("RunsLastElement")); + try (Pipeline pipeline = + new PipelineBuilder().buildFromConfiguration(options)) { + List elements = pipeline.getFlowElements(); + assertTrue(elements.get(elements.size() - 1) + instanceof RunsLastElement); + } + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/WritesAfterProcessingTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/WritesAfterProcessingTests.java new file mode 100644 index 000000000..8d18ad2a0 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/flowelements/WritesAfterProcessingTests.java @@ -0,0 +1,143 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.testclasses.data.TestElementData; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * The check, for the Flow Elements in this module, that processes each one, + * then writes to its Element Data and expects the error. A parallel group, + * {@link ParallelElements}, has no Element Data of its own, so its members' + * Element Data is written to once the group has finished. + * @see Specification + */ +public class WritesAfterProcessingTests { + + private static final Logger LOGGER = + LoggerFactory.getLogger(WritesAfterProcessingTests.class); + + /** + * Writes a value into its own Element Data while it processes. + */ + private static class Member + extends FlowElementBase { + private final String key; + + Member(String key) { + super(LOGGER, (flowData, element) -> + new TestElementData(LOGGER, flowData)); + this.key = key; + } + + @Override + protected void processInternal(FlowData data) { + data.getOrAdd(getTypedDataKey(), getDataFactory()) + .put("written", "during"); + } + + @Override + public String getElementDataKey() { + return key; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist( + Collections.emptyList()); + } + + @Override + public List getProperties() { + return Collections.emptyList(); + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + } + + private interface Write { + void write(); + } + + private static void assertFails(String what, Write write) { + try { + write.write(); + fail(what + " succeeded after the Flow Element had finished"); + } catch (IllegalStateException e) { + // Expected. + } + } + + /** + * Every write method of Element Data fails once its Flow Element has + * finished. + * @param data the Element Data of a finished Flow Element + */ + static void assertWritesFail(ElementData data) { + assertFails("put", () -> data.put("after", "value")); + assertFails("populateFromMap", () -> data.populateFromMap( + Collections.singletonMap("after", "value"))); + } + + @Test + @SuppressWarnings("rawtypes") + public void parallelElements_MembersElementDataRefusesWrites() + throws Exception { + Member first = new Member("first"); + Member second = new Member("second"); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElementsParallel(new FlowElement[] { first, second }) + .build(); + FlowData data = pipeline.createFlowData()) { + data.process(); + + assertEquals(String.valueOf(data.getErrors()), + 0, data.getErrors().size()); + for (Member member : new Member[] { first, second }) { + ElementData own = data.getFromElement(member); + assertEquals("during", own.get("written")); + assertWritesFail(own); + } + } + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/testclasses/flowelements/RunsLastElement.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/testclasses/flowelements/RunsLastElement.java new file mode 100644 index 000000000..f23947c99 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/testclasses/flowelements/RunsLastElement.java @@ -0,0 +1,46 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.testclasses.flowelements; + +import fiftyone.pipeline.core.flowelements.RunsLast; + +/** + * A Flow Element that must be the last Flow Element of a Pipeline and + * outside any parallel group. + */ +public class RunsLastElement extends MultiplyByElement implements RunsLast { + + public RunsLastElement() { + super(1); + } + + @Override + public String getElementDataKey() { + return "runsLast"; + } + + @Override + public String getRunsLastName() { + return "The test element"; + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/testclasses/flowelements/RunsLastElementBuilder.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/testclasses/flowelements/RunsLastElementBuilder.java new file mode 100644 index 000000000..78a0451e2 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/testclasses/flowelements/RunsLastElementBuilder.java @@ -0,0 +1,34 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.testclasses.flowelements; + +import fiftyone.pipeline.annotations.ElementBuilder; +import fiftyone.pipeline.core.flowelements.FlowElement; + +@SuppressWarnings("rawtypes") +@ElementBuilder +public class RunsLastElementBuilder { + public FlowElement build() { + return new RunsLastElement(); + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/typed/TypedKeyMapKeyCaseTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/typed/TypedKeyMapKeyCaseTests.java new file mode 100644 index 000000000..d11442221 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/typed/TypedKeyMapKeyCaseTests.java @@ -0,0 +1,77 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.typed; + +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Keys ignore case, as the specification's access to results asks of + * string keys, in both kinds of map the builder makes. + */ +public class TypedKeyMapKeyCaseTests { + + private static final boolean[] CONCURRENT = {false, true}; + + @Test + public void get_OtherCase_FindsTheValue() { + for (boolean concurrent : CONCURRENT) { + TypedKeyMap map = new TypedKeyMapBuilder(concurrent).build(); + map.put(new TypedKeyDefault<>("device", String.class), "value"); + + assertEquals("value", + map.get(new TypedKeyDefault<>("Device", String.class))); + assertEquals("value", + map.get(new TypedKeyDefault<>("DEVICE", String.class))); + assertTrue(map.containsKey("Device")); + } + } + + @Test + public void put_OtherCase_ReplacesTheValue() { + for (boolean concurrent : CONCURRENT) { + TypedKeyMap map = new TypedKeyMapBuilder(concurrent).build(); + map.put(new TypedKeyDefault<>("device", String.class), "first"); + map.put(new TypedKeyDefault<>("Device", String.class), "second"); + + assertEquals(1, map.getKeys().size()); + assertEquals("second", + map.get(new TypedKeyDefault<>("device", String.class))); + } + } + + @Test + public void getKeys_KeepsTheSpellingFirstGiven() { + for (boolean concurrent : CONCURRENT) { + TypedKeyMap map = new TypedKeyMapBuilder(concurrent).build(); + map.put(new TypedKeyDefault<>("device", String.class), "first"); + map.put(new TypedKeyDefault<>("DEVICE", String.class), "second"); + + assertEquals(Collections.singletonList("device"), map.getKeys()); + } + } +} diff --git a/pipeline/pipeline.core/src/test/resources/options-dotnet-names.xml b/pipeline/pipeline.core/src/test/resources/options-dotnet-names.xml new file mode 100644 index 000000000..75987d234 --- /dev/null +++ b/pipeline/pipeline.core/src/test/resources/options-dotnet-names.xml @@ -0,0 +1,19 @@ + + + + + + MultiplyByElementBuilder + + 8 + + + + + true + false + + diff --git a/pipeline/pipeline.core/src/test/resources/options-test.xml b/pipeline/pipeline.core/src/test/resources/options-test.xml index 335efff11..d02f4f902 100644 --- a/pipeline/pipeline.core/src/test/resources/options-test.xml +++ b/pipeline/pipeline.core/src/test/resources/options-test.xml @@ -27,7 +27,7 @@ http://cloud.51degrees.com/api/v4 + + + 4.0.0 + + com.51degrees + pipeline + 4.5.127-SNAPSHOT + + pipeline.derivedproperty + jar + + 51Degrees :: Pipeline :: Derived Property + The Derived Property Element produces new property values from property values other elements in the Pipeline have already produced, following rules written in a script. + https://github.com/51Degrees/51degrees-java + + + + ${project.groupId} + pipeline.engines + ${project.version} + + + ${project.groupId} + pipeline.core + ${project.version} + + + + org.snakeyaml + snakeyaml-engine + + + org.slf4j + slf4j-api + + + + org.junit.jupiter + junit-jupiter + + + + + + + src/main/resources + + + + ../derived-properties/scripts + fiftyone/pipeline/derivedproperty/scripts + + *.yaml + *.yml + *.json + + + + + diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/BuiltInScript.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/BuiltInScript.java new file mode 100644 index 000000000..5990e24cd --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/BuiltInScript.java @@ -0,0 +1,127 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; + +/** + * Names every script shipped in this package. The scripts are the + * {@code scripts} folder of the derived-properties repository, and each + * constant is named exactly as its script is, so {@link #name()} is the + * script's name. + */ +public enum BuiltInScript { + + /** + * The confidence that the request came from a device being used by a + * human who is viewing the page, being High, Medium, Low or Unknown. + */ + HumanConfidence, + + /** + * Whether the General Data Protection Regulation, or the United + * Kingdom's version of it, is likely to apply to the person making the + * request, judged from ip.CountryCode. + */ + IsGdpr; + + /** + * The folder on the class path the scripts are packaged in. + */ + public static final String RESOURCE_FOLDER = + "/fiftyone/pipeline/derivedproperty/scripts/"; + + /** + * The file extensions a script is read from, in the order tried. + */ + private static final String[] EXTENSIONS = {".yaml", ".yml", ".json"}; + + /** + * The size of each read from the packaged script. + */ + private static final int BUFFER_BYTES = 8192; + + /** + * The text of the script, read from the package. + * @return the text, or null where the package was built without the + * script + */ + public String readText() { + for (String extension : EXTENSIONS) { + try (InputStream stream = BuiltInScript.class.getResourceAsStream( + RESOURCE_FOLDER + name() + extension)) { + if (stream != null) { + return readAll(stream); + } + } catch (IOException e) { + throw new UncheckedIOException( + "The built-in script '" + name() + "' cannot be read.", + e); + } + } + return null; + } + + /** + * The built-in script with the name given, matched exactly. + * @param name the script's name + * @return the script, or null where no built-in script has the name + */ + public static BuiltInScript fromName(String name) { + for (BuiltInScript script : values()) { + if (script.name().equals(name)) { + return script; + } + } + return null; + } + + /** + * Every built-in script's name, for a message. + * @return the names, separated by a comma and a space + */ + public static String names() { + StringBuilder names = new StringBuilder(); + for (BuiltInScript script : values()) { + if (names.length() > 0) { + names.append(", "); + } + names.append(script.name()); + } + return names.toString(); + } + + private static String readAll(InputStream stream) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + byte[] buffer = new byte[BUFFER_BYTES]; + int read; + while ((read = stream.read(buffer)) > 0) { + bytes.write(buffer, 0, read); + } + return new String(bytes.toByteArray(), StandardCharsets.UTF_8); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/DerivedScriptFault.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/DerivedScriptFault.java new file mode 100644 index 000000000..0b5bdc6a0 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/DerivedScriptFault.java @@ -0,0 +1,112 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty; + +import java.io.Serializable; + +/** + * One thing wrong with a derived property script. + */ +public final class DerivedScriptFault implements Serializable { + + private static final long serialVersionUID = 1L; + + private final String script; + private final String source; + private final String path; + private final int line; + private final String message; + + /** + * Construct a fault. + * @param script the name the script was given, or empty where it had + * none + * @param source the built-in script's name, the file path, or "code" + * @param path the place in the document, such as Rules[3].When.All[1], + * or empty for the document as a whole + * @param line the one based line, or 0 where the parser gave none + * @param message what is wrong + */ + public DerivedScriptFault( + String script, + String source, + String path, + int line, + String message) { + this.script = script == null ? "" : script; + this.source = source == null ? "" : source; + this.path = path == null ? "" : path; + this.line = line; + this.message = message; + } + + /** + * @return the name the script was given, or empty where it had none + */ + public String getScript() { + return script; + } + + /** + * @return the built-in script's name, the file path, or "code" + */ + public String getSource() { + return source; + } + + /** + * @return the place in the document, such as Rules[3].When.All[1], or + * empty for the document as a whole + */ + public String getPath() { + return path; + } + + /** + * @return the one based line, or 0 where the parser gave none + */ + public int getLine() { + return line; + } + + /** + * @return what is wrong + */ + public String getMessage() { + return message; + } + + /** + * The fault as one line, giving the script, the source, the line where + * known, the path and the message. + * @return the fault as one line + */ + @Override + public String toString() { + return (script.isEmpty() ? "script" : script) + + " (" + source + ")" + + (line > 0 ? " line " + line : "") + + " at " + (path.isEmpty() ? "(document)" : path) + + ": " + message; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/DerivedScriptValidationException.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/DerivedScriptValidationException.java new file mode 100644 index 000000000..3b0681342 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/DerivedScriptValidationException.java @@ -0,0 +1,66 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * Every fault found in the scripts given to a Derived Property Element + * builder, raised once after every script has been checked. The message + * lists every fault, one per line. + */ +public class DerivedScriptValidationException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private final DerivedScriptFault[] faults; + + /** + * Construct the exception. + * @param faults every fault found + */ + public DerivedScriptValidationException(List faults) { + super(message(faults)); + this.faults = faults.toArray(new DerivedScriptFault[0]); + } + + /** + * @return every fault found, in the order found + */ + public List getFaults() { + return Collections.unmodifiableList(Arrays.asList(faults)); + } + + private static String message(List faults) { + StringBuilder message = new StringBuilder(); + for (DerivedScriptFault fault : faults) { + if (message.length() > 0) { + message.append('\n'); + } + message.append(fault); + } + return message.toString(); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/data/DerivedPropertyData.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/data/DerivedPropertyData.java new file mode 100644 index 000000000..8531906d5 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/data/DerivedPropertyData.java @@ -0,0 +1,86 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.data; + +import fiftyone.pipeline.core.data.ElementDataBase; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyReason; +import org.slf4j.Logger; + +import java.util.concurrent.ConcurrentSkipListMap; + +/** + * The Element Data of the Derived Property Elements. Every Derived Property + * Element in a Pipeline writes into the one instance on a request, so two of + * them must not be members of the same parallel group. Whoever builds the + * Pipeline keeps them apart, as nothing checks it. The values are held in a + * map whose keys are matched without regard to case. + *

    + * See the + * Derived + * Property Element specification. + */ +public class DerivedPropertyData + extends ElementDataBase + implements IDerivedPropertyData { + + /** + * Construct the Element Data for one request. + * @param logger used for logging + * @param flowData the request the data belongs to + */ + public DerivedPropertyData(Logger logger, FlowData flowData) { + super( + logger, + flowData, + new ConcurrentSkipListMap( + String.CASE_INSENSITIVE_ORDER)); + } + + @Override + @SuppressWarnings("unchecked") + public AspectPropertyValue getAs( + String propertyName, + Class valueType) { + Object value = get(propertyName); + if (value instanceof AspectPropertyValue == false) { + throw new PropertyMissingException( + MissingPropertyReason.Unknown, + propertyName, + "There is no derived property '" + propertyName + "'. A " + + "derived property is named by the Output.Name of a " + + "script the Derived Property Element holds."); + } + AspectPropertyValue result = (AspectPropertyValue) value; + if (result.hasValue() && + valueType.isInstance(result.getValue()) == false) { + throw new ClassCastException( + "The derived property '" + propertyName + "' holds a " + + result.getValue().getClass().getSimpleName() + + ", not a " + valueType.getSimpleName() + "."); + } + return (AspectPropertyValue) result; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/data/IDerivedPropertyData.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/data/IDerivedPropertyData.java new file mode 100644 index 000000000..561d50a7e --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/data/IDerivedPropertyData.java @@ -0,0 +1,50 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.data; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.engines.data.AspectPropertyValue; + +/** + * The Element Data every Derived Property Element writes under the key + * {@code derived}. It holds one {@link AspectPropertyValue} per script, + * keyed by the script's output property name, matched without regard to + * case. + */ +public interface IDerivedPropertyData extends ElementData { + + /** + * Get a derived property's value. A value that has no value carries a + * message naming every source property that could not be read, and + * {@link AspectPropertyValue#getValue()} then throws with that message. + * @param propertyName the output property name of a script + * @param valueType the type the script gives, being {@link String}, + * {@link Boolean}, {@link Integer} or {@link Double} + * @param the value type + * @return the value + * @throws fiftyone.pipeline.engines.exceptions.PropertyMissingException + * where no script produces the property + * @throws ClassCastException where the value is not of the type given + */ + AspectPropertyValue getAs(String propertyName, Class valueType); +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/CanonicalJson.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/CanonicalJson.java new file mode 100644 index 000000000..721690a99 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/CanonicalJson.java @@ -0,0 +1,302 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Prints a model as canonical JSON, with PascalCase keys, two space indent, + * literal types preserved, and the inferred types and computed dependencies + * included, as tools/canonical.mjs in the derived-properties repository + * does. The debug log shows this text, so anyone holding the log can + * reconstruct what was evaluated, and a YAML script and the JSON that + * mirrors it print the same. + */ +final class CanonicalJson { + + private static final String INDENT = " "; + + /** + * JSON text written as it is. + */ + private static final class Raw { + private final String text; + + Raw(String text) { + this.text = text; + } + } + + private CanonicalJson() { + } + + /** + * Print a model. + * @param script the model + * @return the canonical JSON, without a trailing new line + */ + static String print(DerivedScript script) { + StringBuilder text = new StringBuilder(); + write(text, object(script), 0); + return text.toString(); + } + + private static Map object(DerivedScript script) { + Map result = new LinkedHashMap<>(); + result.put("Format", new Raw("1")); + result.put("Name", script.getName()); + result.put("Version", script.getVersion()); + if (script.isDeprecated()) { + result.put("Deprecated", Boolean.TRUE); + result.put("DeprecationNote", script.getDeprecationNote()); + } + result.put("Output", output(script.getOutput())); + + // The inferred type of every source property, which the file does + // not carry because the literals compared with it give it. + Map properties = new LinkedHashMap<>(); + for (SourceProperty property : script.getProperties()) { + Map type = new LinkedHashMap<>(); + type.put("Type", property.getType().toString()); + properties.put(property.getName(), type); + } + result.put("Properties", properties); + + if (script.getChecks().isEmpty() == false) { + Map checks = new LinkedHashMap<>(); + for (DerivedScript.NamedCheck check : script.getChecks()) { + checks.put(check.getName(), condition(check.getCondition())); + } + result.put("Checks", checks); + } + List rules = new ArrayList<>(); + for (DerivedScript.Rule rule : script.getRules()) { + Map entry = new LinkedHashMap<>(); + if (rule.getWhen() == null) { + entry.put("Else", literal(rule.getValue())); + } else { + entry.put("When", condition(rule.getWhen())); + entry.put("Then", literal(rule.getValue())); + } + rules.add(entry); + } + result.put("Rules", rules); + return result; + } + + private static Map output(OutputModel output) { + Map result = new LinkedHashMap<>(); + result.put("Name", output.getName()); + if (output.getDescription().isEmpty() == false) { + result.put("Description", output.getDescription()); + } + if (output.getValueType() != null) { + result.put("ValueType", output.getValueType().toString()); + } + putIfGiven(result, "StoredValueType", output.getStoredValueType()); + putIfGiven(result, "DefaultValue", output.getDefaultValue()); + result.put("IsList", output.isList()); + putIfGiven(result, "IsMandatory", output.getIsMandatory()); + putIfGiven(result, "IsObsolete", output.getIsObsolete()); + putIfGiven(result, "Category", output.getCategory()); + putIfGiven(result, "IsPopular", output.getIsPopular()); + putIfGiven(result, "ExportValues", output.getExportValues()); + putIfGiven(result, "Url", output.getUrl()); + if (output.getDisplayOrder() != null) { + result.put("DisplayOrder", + new Raw(JsText.number(output.getDisplayOrder()))); + } + if (output.getPropertyId() != null) { + result.put("PropertyId", + new Raw(JsText.number(output.getPropertyId()))); + } + if (output.getVendorIds() != null) { + result.put("VendorIds", node(output.getVendorIds())); + } + if (output.getDependencies() != null) { + result.put("Dependencies", node(output.getDependencies())); + } else { + result.put("Dependencies", + new ArrayList(output.getComputed())); + } + if (output.getValues() != null) { + List values = new ArrayList<>(); + for (OutputModel.ValueEntry value : output.getValues()) { + Map entry = new LinkedHashMap<>(); + entry.put("Name", literal(value.getName())); + if (value.getDescription() != null) { + entry.put("Description", value.getDescription()); + } + values.add(entry); + } + result.put("Values", values); + } + return result; + } + + private static void putIfGiven( + Map result, + String key, + Object value) { + if (value != null) { + result.put(key, value); + } + } + + private static Map condition(Condition condition) { + Map result = new LinkedHashMap<>(); + switch (condition.getKind()) { + case COMPARE: + result.put("Property", condition.getProperty()); + result.put(condition.getOperator().toString(), + node(condition.getOperand())); + break; + case CHECK: + result.put("Check", condition.getCheck()); + break; + case AGGREGATE: + result.put( + condition.isCountPassed() ? "Passed" : "Failed", + condition.getGroupNames() == null ? + "Checks" : + new ArrayList(condition.getGroupNames())); + result.put(condition.getOperator().toString(), + new Raw(JsText.number(condition.getCount()))); + break; + case ALL: + case ANY: + List items = new ArrayList<>(); + for (int i = 0; i < condition.itemCount(); i++) { + items.add(condition(condition.item(i))); + } + result.put( + condition.getKind() == Condition.Kind.ALL ? "All" : "Any", + items); + break; + case NOT: + result.put("Not", condition(condition.item(0))); + break; + default: + break; + } + return result; + } + + private static Object literal(Literal literal) { + switch (literal.getKind()) { + case BOOL: + return literal.getBool(); + case NUMBER: + return new Raw(JsText.number(literal.getNumber())); + default: + return literal.getText(); + } + } + + /** + * A value from the script with its types as written. + */ + private static Object node(ScriptNode node) { + switch (node.getKind()) { + case NULL: + case BOOL: + case NUMBER: + return new Raw(DerivedScriptValidator.jsonOf(node)); + case STRING: + return node.getText(); + case LIST: + List items = new ArrayList<>(); + for (ScriptNode item : node.getItems()) { + items.add(node(item)); + } + return items; + default: + Map result = new LinkedHashMap<>(); + for (int i = 0; i < node.getKeys().size(); i++) { + result.put( + node.getKeys().get(i), + node(node.getValues().get(i))); + } + return result; + } + } + + /** + * Write a value as JSON.stringify does with an indent of two. + */ + private static void write(StringBuilder text, Object value, int depth) { + if (value instanceof Raw) { + text.append(((Raw) value).text); + } else if (value instanceof String) { + text.append(JsText.quote((String) value)); + } else if (value instanceof Boolean) { + text.append(((Boolean) value) ? "true" : "false"); + } else if (value instanceof List) { + List items = (List) value; + if (items.isEmpty()) { + text.append("[]"); + return; + } + text.append('['); + for (int i = 0; i < items.size(); i++) { + if (i > 0) { + text.append(','); + } + newLine(text, depth + 1); + write(text, items.get(i), depth + 1); + } + newLine(text, depth); + text.append(']'); + } else if (value instanceof Map) { + Map entries = (Map) value; + if (entries.isEmpty()) { + text.append("{}"); + return; + } + text.append('{'); + boolean first = true; + for (Map.Entry entry : entries.entrySet()) { + if (first == false) { + text.append(','); + } + first = false; + newLine(text, depth + 1); + text.append(JsText.quote((String) entry.getKey())) + .append(": "); + write(text, entry.getValue(), depth + 1); + } + newLine(text, depth); + text.append('}'); + } + } + + private static void newLine(StringBuilder text, int depth) { + text.append('\n'); + for (int i = 0; i < depth; i++) { + text.append(INDENT); + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/CompiledScript.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/CompiledScript.java new file mode 100644 index 000000000..31e0d7f23 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/CompiledScript.java @@ -0,0 +1,71 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A script with each of its source properties resolved to one of the + * element's reads. + */ +final class CompiledScript { + + private final DerivedScript model; + private final int[] reads; + + /** + * Records, without a lock, that the error for a request with no Element + * Data under a key the script reads has been logged. It is the only + * state shared between requests. + */ + private final AtomicBoolean logged = new AtomicBoolean(false); + + /** + * @param model the validated script + * @param reads for each of the script's source properties, the index of + * the element's read that supplies it + */ + CompiledScript(DerivedScript model, int[] reads) { + this.model = model; + this.reads = reads.clone(); + } + + DerivedScript getModel() { + return model; + } + + int readCount() { + return reads.length; + } + + int read(int i) { + return reads[i]; + } + + /** + * @return true the first time it is called, and false after that + */ + boolean firstMissingData() { + return logged.compareAndSet(false, true); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Condition.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Condition.java new file mode 100644 index 000000000..a3edd1ee0 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Condition.java @@ -0,0 +1,340 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.Collections; +import java.util.List; + +/** + * One condition of a script, which is always true or false, because a + * script only runs once every source property it names has been read. + */ +final class Condition { + + /** + * The forms a condition takes. + */ + enum Kind { COMPARE, CHECK, AGGREGATE, ALL, ANY, NOT } + + private final Kind kind; + + // A comparison of a source property with a literal. + private final int slot; + private final String property; + private final Operator operator; + private final ScriptNode operand; + private final ValueType type; + private final Literal literal; + private final Literal[] literals; + + // A reference to a check by name. + private final String check; + private final int index; + + // A count of the checks in a group that passed or failed. A null group + // is every check. + private final boolean countPassed; + private final int[] group; + private final List groupNames; + private final long count; + + // The members of All and Any, or the one condition of Not. + private final Condition[] items; + + private Condition( + Kind kind, + int slot, + String property, + Operator operator, + ScriptNode operand, + ValueType type, + Literal literal, + Literal[] literals, + String check, + int index, + boolean countPassed, + int[] group, + List groupNames, + long count, + Condition[] items) { + this.kind = kind; + this.slot = slot; + this.property = property; + this.operator = operator; + this.operand = operand; + this.type = type; + this.literal = literal; + this.literals = literals; + this.check = check; + this.index = index; + this.countPassed = countPassed; + this.group = group; + this.groupNames = groupNames; + this.count = count; + this.items = items; + } + + /** + * What a condition that could not be read becomes. A fault has been + * recorded by then, so the model is discarded unevaluated. + */ + static Condition faulted() { + return compound(Kind.ALL, new Condition[0]); + } + + static Condition comparison( + int slot, + String property, + Operator operator, + ScriptNode operand, + ValueType type) { + Literal literal = Literal.NONE; + Literal[] literals = new Literal[0]; + if (operand.getKind() == ScriptNode.Kind.LIST) { + literals = new Literal[operand.getItems().size()]; + for (int i = 0; i < literals.length; i++) { + literals[i] = Literal.of(operand.getItems().get(i)); + } + } else { + literal = Literal.of(operand); + } + return new Condition(Kind.COMPARE, slot, property, operator, operand, + type, literal, literals, null, -1, false, null, null, 0, null); + } + + static Condition checkReference(String check, int index) { + return new Condition(Kind.CHECK, -1, null, null, null, null, + Literal.NONE, null, check, index, false, null, null, 0, null); + } + + /** + * @param countPassed true for Passed, false for Failed + * @param group the indexes of the checks counted, or null for every + * check + * @param groupNames the names of the checks counted, or null for every + * check + * @param operator the comparison + * @param operand the whole number compared with, as written + */ + static Condition aggregate( + boolean countPassed, + int[] group, + List groupNames, + Operator operator, + ScriptNode operand) { + return new Condition(Kind.AGGREGATE, -1, null, operator, operand, + ValueType.INT, Literal.NONE, null, null, -1, countPassed, group, + groupNames == null ? + null : Collections.unmodifiableList(groupNames), + (long) operand.getNumber(), null); + } + + /** + * @param kind ALL, ANY or NOT + * @param items the members, or the one condition of Not + */ + static Condition compound(Kind kind, Condition[] items) { + return new Condition(kind, -1, null, null, null, null, Literal.NONE, + null, null, -1, false, null, null, 0, items); + } + + Kind getKind() { + return kind; + } + + String getProperty() { + return property; + } + + Operator getOperator() { + return operator; + } + + /** + * @return the literal compared with, as written in the script + */ + ScriptNode getOperand() { + return operand; + } + + String getCheck() { + return check; + } + + boolean isCountPassed() { + return countPassed; + } + + /** + * @return the names of the checks counted, or null for every check + */ + List getGroupNames() { + return groupNames; + } + + long getCount() { + return count; + } + + /** + * @return the number of members of All or Any, or 1 for Not + */ + int itemCount() { + return items.length; + } + + Condition item(int i) { + return items[i]; + } + + /** + * Evaluate the condition. Every slot is available. + * @param slots the source properties, read for this script + * @param checks the result of every check, in the order defined + * @return true or false + */ + boolean test(Slot[] slots, boolean[] checks) { + switch (kind) { + case COMPARE: + return compare(slots[slot]); + case CHECK: + return checks[index]; + case AGGREGATE: + return compareCount(countOf(checks)); + case ALL: + for (Condition item : items) { + if (item.test(slots, checks) == false) { + return false; + } + } + return true; + case ANY: + for (Condition item : items) { + if (item.test(slots, checks)) { + return true; + } + } + return false; + case NOT: + return items[0].test(slots, checks) == false; + default: + return false; + } + } + + private long countOf(boolean[] checks) { + long counted = 0; + if (group == null) { + for (boolean passed : checks) { + if (passed == countPassed) { + counted++; + } + } + } else { + for (int member : group) { + if (checks[member] == countPassed) { + counted++; + } + } + } + return counted; + } + + private boolean compareCount(long counted) { + switch (operator) { + case EQ: + return counted == count; + case NE: + return counted != count; + case GT: + return counted > count; + case GE: + return counted >= count; + case LT: + return counted < count; + case LE: + return counted <= count; + default: + return false; + } + } + + /** + * Apply the operator to the value read for the property. + */ + private boolean compare(Slot value) { + switch (operator) { + case IN: + for (Literal member : literals) { + if (equal(value, member)) { + return true; + } + } + return false; + case NOT_IN: + for (Literal member : literals) { + if (equal(value, member)) { + return false; + } + } + return true; + case EQ: + return equal(value, literal); + case NE: + return equal(value, literal) == false; + case STARTS_WITH: + return value.getString().startsWith(literal.getText()); + case ENDS_WITH: + return value.getString().endsWith(literal.getText()); + case CONTAINS: + return value.getString().contains(literal.getText()); + case GT: + return value.number(type) > literal.getNumber(); + case GE: + return value.number(type) >= literal.getNumber(); + case LT: + return value.number(type) < literal.getNumber(); + case LE: + return value.number(type) <= literal.getNumber(); + default: + return false; + } + } + + /** + * Whether the value read equals a literal, ordinally for strings. + */ + private boolean equal(Slot value, Literal other) { + switch (type) { + case BOOL: + return other.getKind() == ScriptNode.Kind.BOOL && + value.getBool() == other.getBool(); + case STRING: + return other.getKind() == ScriptNode.Kind.STRING && + value.getString().equals(other.getText()); + default: + return other.getKind() == ScriptNode.Kind.NUMBER && + value.number(type) == other.getNumber(); + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedOutputMetaData.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedOutputMetaData.java new file mode 100644 index 000000000..73a03b80b --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedOutputMetaData.java @@ -0,0 +1,204 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The Output block of a script, with the field names 51Degrees uses for the + * metadata of every property. It is the whole definition of the derived + * property, so a JSON builder, the cloud or a documentation generator can + * read every field. A field the script does not give is null. + */ +public final class DerivedOutputMetaData { + + private final OutputModel output; + private final List vendorIds; + private final List dependencies; + private final List values; + + DerivedOutputMetaData(OutputModel output) { + this.output = output; + this.vendorIds = output.getVendorIds() == null ? + null : plainList(output.getVendorIds()); + this.dependencies = output.getDependencies() == null ? + Collections.unmodifiableList( + new ArrayList(output.getComputed())) : + plainList(output.getDependencies()); + if (output.getValues() == null) { + this.values = null; + } else { + List entries = new ArrayList<>(); + for (OutputModel.ValueEntry entry : output.getValues()) { + entries.add(new DerivedValueMetaData( + entry.getName().toString(), entry.getDescription())); + } + this.values = Collections.unmodifiableList(entries); + } + } + + /** + * @return the property name published under the derived Element Data + * key + */ + public String getName() { + return output.getName(); + } + + /** + * @return what the property asserts + */ + public String getDescription() { + return output.getDescription(); + } + + /** + * @return string, bool, int or double + */ + public String getValueType() { + return output.getValueType().toString(); + } + + /** + * @return the Java type of the value, being {@link String}, + * {@link Boolean}, {@link Integer} or {@link Double} + */ + public Class getValueClass() { + return output.getValueType().getValueClass(); + } + + public String getStoredValueType() { + return output.getStoredValueType(); + } + + /** + * @return the string form of the default recorded for the property. + * This is metadata only, and nothing reads it while a request is + * processed. + */ + public String getDefaultValue() { + return output.getDefaultValue(); + } + + /** + * @return false, as format 1 has no list outputs + */ + public boolean isList() { + return output.isList(); + } + + public Boolean getIsMandatory() { + return output.getIsMandatory(); + } + + public Boolean getIsObsolete() { + return output.getIsObsolete(); + } + + public String getCategory() { + return output.getCategory(); + } + + public Boolean getIsPopular() { + return output.getIsPopular(); + } + + public Boolean getExportValues() { + return output.getExportValues(); + } + + public String getUrl() { + return output.getUrl(); + } + + public Long getDisplayOrder() { + return output.getDisplayOrder(); + } + + public Long getPropertyId() { + return output.getPropertyId(); + } + + /** + * @return the list as written, with numbers as {@link Double} + */ + public List getVendorIds() { + return vendorIds; + } + + /** + * @return the list the script gives, or every source property its + * checks and rules name, in elementKey.PropertyName form, where it gives + * none + */ + public List getDependencies() { + return dependencies; + } + + /** + * @return the values the property can have, or null where the script + * gives no list + */ + public List getValues() { + return values; + } + + /** + * A list from the script as plain Java values, being null, a + * {@link Boolean}, a {@link Double}, a {@link String}, a list or a map. + */ + private static List plainList(ScriptNode node) { + List items = new ArrayList<>(); + for (ScriptNode item : node.getItems()) { + items.add(plain(item)); + } + return Collections.unmodifiableList(items); + } + + private static Object plain(ScriptNode node) { + switch (node.getKind()) { + case BOOL: + return node.getBool(); + case NUMBER: + return node.getNumber(); + case STRING: + return node.getText(); + case LIST: + return plainList(node); + case MAP: + Map values = new LinkedHashMap<>(); + for (int i = 0; i < node.getKeys().size(); i++) { + values.put( + node.getKeys().get(i), + plain(node.getValues().get(i))); + } + return Collections.unmodifiableMap(values); + default: + return null; + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElement.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElement.java new file mode 100644 index 000000000..98d0e47d5 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElement.java @@ -0,0 +1,570 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.ElementPropertyMetaDataDefault; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.TryGetResult; +import fiftyone.pipeline.core.data.factories.ElementDataFactory; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.FlowElementBase; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.core.typed.TypedKeyDefault; +import fiftyone.pipeline.derivedproperty.data.IDerivedPropertyData; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +/** + * The Derived Property Element, which produces new property values from + * property values other Flow Elements have already produced, following + * rules written in a script, and writes them into the Element Data under + * the key {@code derived}, which every instance in a Pipeline shares. + *

    + * The element holds no data file, makes no request over the network and + * reads no evidence. Everything it holds is fixed when it is built, so + * concurrent requests need nothing further, and per request it reads each + * source property once, then runs each script. + *

    + * Every Derived Property Element in a Pipeline writes into the same Element + * Data, so two of them must not be members of the same parallel group. + * Whoever builds the Pipeline keeps them apart, as nothing checks it. + *

    + * See the + * Derived + * Property Element specification. + */ +public final class DerivedPropertyElement + extends FlowElementBase + implements IDerivedPropertyElement { + + /** + * The Element Data key every Derived Property Element in a Pipeline + * writes under. + */ + public static final String ELEMENT_DATA_KEY = "derived"; + + /** + * The sentence that closes every message for a derived property with no + * value, being USUAL_CAUSES in tools/evaluate.mjs. + */ + static final String USUAL_CAUSES = + "Usual causes are the element that supplies the property not being " + + "in the pipeline, or being added after this element rather than " + + "before it, the property being excluded in the engine " + + "configuration, the property not being included in the resource " + + "key, or JavaScript that populates the property not having run yet."; + + private static final TypedKey DATA_KEY = + new TypedKeyDefault<>(ELEMENT_DATA_KEY, IDerivedPropertyData.class); + + private final CompiledScript[] scripts; + private final SourceRead[] reads; + private final List> dataKeys; + private final List properties; + private final List metadata; + private final EvidenceKeyFilter evidenceKeyFilter = + new EvidenceKeyFilterWhitelist(Collections.emptyList()); + + /** + * Construct the element from validated scripts. Use + * {@link DerivedPropertyElementBuilder} to build one. + * @param logger used for logging + * @param dataFactory makes the Element Data + * @param models the validated scripts + */ + DerivedPropertyElement( + Logger logger, + ElementDataFactory dataFactory, + List models) { + super(logger, dataFactory); + Map keyIndex = new HashMap<>(); + Map readIndex = new HashMap<>(); + List> keys = new ArrayList<>(); + List allReads = new ArrayList<>(); + List compiled = new ArrayList<>(); + List outputs = new ArrayList<>(); + List scriptData = new ArrayList<>(); + for (DerivedScript model : models) { + int[] scriptReads = new int[model.getProperties().size()]; + for (int i = 0; i < scriptReads.length; i++) { + SourceProperty property = model.getProperties().get(i); + String id = property.getKey() + "|" + property.getType(); + Integer index = readIndex.get(id); + if (index == null) { + int key = -1; + if (property.getElementKey().equals(ELEMENT_DATA_KEY) == + false) { + Integer known = keyIndex.get(property.getElementKey()); + if (known == null) { + known = keys.size(); + keyIndex.put(property.getElementKey(), known); + keys.add(new TypedKeyDefault( + property.getElementKey(), ElementData.class)); + } + key = known; + } + index = allReads.size(); + readIndex.put(id, index); + allReads.add(new SourceRead(property, key)); + } + scriptReads[i] = index; + } + compiled.add(new CompiledScript(model, scriptReads)); + scriptData.add(new DerivedPropertyMetaData(model)); + OutputModel output = model.getOutput(); + outputs.add(new ElementPropertyMetaDataDefault( + output.getName(), + this, + output.getCategory() == null ? "" : output.getCategory(), + output.getValueType().getValueClass(), + true)); + } + this.scripts = compiled.toArray(new CompiledScript[0]); + this.reads = allReads.toArray(new SourceRead[0]); + this.dataKeys = Collections.unmodifiableList(keys); + this.properties = Collections.unmodifiableList(outputs); + this.metadata = Collections.unmodifiableList(scriptData); + } + + @Override + public String getElementDataKey() { + return ELEMENT_DATA_KEY; + } + + @Override + public TypedKey getTypedDataKey() { + return DATA_KEY; + } + + /** + * The element reads properties other elements produce and nothing from + * the request itself, so it accepts no evidence. + * @return a filter that includes no key + */ + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return evidenceKeyFilter; + } + + /** + * @return one entry per script, with the output property's name, the + * Java type of its value, the category from the script's output block, + * and available true + */ + @Override + public List getProperties() { + return properties; + } + + @Override + public List getScripts() { + return metadata; + } + + /** + * Every source property the element reads from another element's + * Element Data, once each. + */ + List sourceProperties() { + List result = new ArrayList<>(); + for (SourceRead read : reads) { + if (read.getKey() >= 0) { + result.add(read.getProperty()); + } + } + return result; + } + + /** + * Read each source property once, then run each script, writing one + * value, or one value with no value carrying a message, per script. + */ + @Override + protected void processInternal(FlowData data) { + IDerivedPropertyData out = data.getOrAdd(DATA_KEY, getDataFactory()); + ElementData[] sources = new ElementData[dataKeys.size()]; + for (int i = 0; i < sources.length; i++) { + TryGetResult source = + data.tryGetValue(dataKeys.get(i)); + if (source.hasValue()) { + sources[i] = source.getValue(); + } + } + Slot[] slots = new Slot[reads.length]; + for (int i = 0; i < reads.length; i++) { + if (reads[i].getKey() >= 0) { + slots[i] = SourceValues.read( + sources[reads[i].getKey()], reads[i].getProperty()); + } + } + for (CompiledScript script : scripts) { + processScript(script, slots, sources, out); + } + } + + private void processScript( + CompiledScript script, + Slot[] slots, + ElementData[] sources, + IDerivedPropertyData out) { + DerivedScript model = script.getModel(); + Slot[] scriptSlots = new Slot[script.readCount()]; + List missing = null; + List reasons = null; + SourceRead noData = null; + for (int i = 0; i < scriptSlots.length; i++) { + SourceRead read = reads[script.read(i)]; + if (read.getKey() < 0) { + scriptSlots[i] = SourceValues.read(out, read.getProperty()); + } else { + scriptSlots[i] = slots[script.read(i)]; + if (sources[read.getKey()] == null && noData == null) { + noData = read; + } + } + if (scriptSlots[i].isAvailable() == false) { + if (missing == null) { + missing = new ArrayList<>(); + reasons = new ArrayList<>(); + } + missing.add(model.getProperties().get(i).getName()); + reasons.add(scriptSlots[i].getReason()); + } + } + OutputModel output = model.getOutput(); + if (missing != null) { + if (noData != null && script.firstMissingData()) { + String key = noData.getProperty().getElementKey(); + logger.error("Derived property script '" + model.getName() + + "' reads '" + noData.getProperty().getName() + "', and a " + + "request carried no element data under '" + key + "'. " + + "Add this element to the pipeline after the element " + + "that supplies '" + key + "'. This error is reported " + + "once for this script."); + } + AspectPropertyValueDefault value = + new AspectPropertyValueDefault<>(); + value.setNoValueMessage( + missingMessage(output.getName(), missing, reasons)); + out.put(output.getName(), value); + return; + } + out.put(output.getName(), + valueOf(output.getValueType(), model.evaluate(scriptSlots))); + } + + private static AspectPropertyValue valueOf( + ValueType type, + Literal literal) { + switch (type) { + case BOOL: + return new AspectPropertyValueDefault<>(literal.getBool()); + case INT: + return new AspectPropertyValueDefault<>( + (int) literal.getNumber()); + case DOUBLE: + return new AspectPropertyValueDefault<>(literal.getNumber()); + default: + return new AspectPropertyValueDefault<>(literal.getText()); + } + } + + /** + * Say why a script has no value, naming every source property that + * could not be read, in the wording of missingMessage in + * tools/evaluate.mjs. + */ + static String missingMessage( + String outputName, + List missing, + List reasons) { + StringBuilder message = new StringBuilder() + .append("Derived property '").append(outputName) + .append("' has no value because "); + if (missing.size() == 1) { + message.append("1 source property was not available."); + } else { + message.append(missing.size()) + .append(" source properties were not available."); + } + for (int i = 0; i < missing.size(); i++) { + message.append(" '").append(missing.get(i)).append("' (") + .append(reasons.get(i)).append(")."); + } + return message.append(' ').append(USUAL_CAUSES).toString(); + } + + /** + * Check that some element in the Pipeline supplies every source + * property a script names, and that no two Derived Property Elements + * produce the same property. Where an element sits is not judged, + * because a Pipeline lists the members of a parallel group after the + * elements at the top level, so a position does not say what ran + * first. + *

    + * See the + * Pipeline + * check. + * @param pipeline the Pipeline the element is being added to + * @throws PipelineConfigurationException where the check fails + */ + @Override + public void addPipeline(Pipeline pipeline) { + if (pipeline != null) { + checkPipeline(pipeline); + } + super.addPipeline(pipeline); + } + + private void checkPipeline(Pipeline pipeline) { + Map> available = + availableProperties(pipeline); + List> elements = elementsOf(pipeline, available); + Set keys = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + keys.addAll(available.keySet()); + boolean allLoaded = allLoaded(pipeline); + List problems = new ArrayList<>(); + for (CompiledScript script : scripts) { + for (SourceProperty property : + script.getModel().getProperties()) { + String problem = checkSupplied(elements, keys, allLoaded, + script.getModel().getName(), property); + if (problem != null) { + problems.add(problem); + } + } + } + // Each name counts the distinct Derived Property Elements that + // produce it. + Map producers = new HashMap<>(); + for (FlowElement element : elements) { + if (element instanceof DerivedPropertyElement) { + for (CompiledScript script : + ((DerivedPropertyElement) element).scripts) { + String name = script.getModel().getOutput().getName() + .toLowerCase(Locale.ROOT); + Integer count = producers.get(name); + producers.put(name, count == null ? 1 : count + 1); + } + } + } + for (CompiledScript script : scripts) { + String name = script.getModel().getOutput().getName(); + Integer count = producers.get(name.toLowerCase(Locale.ROOT)); + if (count != null && count > 1) { + problems.add("two Derived Property Elements in the pipeline " + + "produce the derived property '" + name + "'"); + } + } + if (problems.isEmpty() == false) { + throw new PipelineConfigurationException( + String.join(". ", problems)); + } + } + + /** + * Why no element supplies a source property, or null where one does. + * An element whose properties are not known yet, such as a cloud engine + * that could not reach the service, is taken to supply the properties + * under its key, with a warning, so that a passing failure does not + * stop the Pipeline being built. + */ + private String checkSupplied( + List> elements, + Set keys, + boolean allLoaded, + String script, + SourceProperty property) { + String key = property.getElementKey(); + String name = property.getPropertyName(); + boolean inspected = false; + for (FlowElement element : elements) { + if (element.getElementDataKey().equalsIgnoreCase(key) == false) { + continue; + } + inspected = true; + if (lists(element, name)) { + return null; + } + if (element.hasLoadedProperties() == false) { + warnNotLoaded(script, property); + return null; + } + } + if (inspected == false && keys.contains(key) == false) { + return "derived property script '" + script + "' reads '" + + property.getName() + "', and no element in the pipeline has " + + "the element data key '" + key + "'"; + } + if (inspected == false && allLoaded == false) { + warnNotLoaded(script, property); + return null; + } + return "derived property script '" + script + "' reads '" + + property.getName() + "', and no element under '" + key + + "' lists the property '" + name + "'"; + } + + private void warnNotLoaded(String script, SourceProperty property) { + logger.warn("Derived property script '" + script + "' reads '" + + property.getName() + "', and the element under '" + + property.getElementKey() + "' has not loaded its properties " + + "yet, so whether it supplies '" + property.getPropertyName() + + "' cannot be checked."); + } + + /** + * Whether an element lists a property in its metadata, matched without + * regard to case. + */ + private static boolean lists(FlowElement element, String name) { + try { + for (ElementPropertyMetaData property : element.getProperties()) { + if (property.getName().equalsIgnoreCase(name)) { + return true; + } + } + } catch (RuntimeException e) { + return false; + } + return false; + } + + /** + * The Pipeline's available properties by Element Data key, which covers + * the members of every parallel group as well as the elements at the + * top level. + */ + private static Map> + availableProperties(Pipeline pipeline) { + try { + return pipeline.getElementAvailableProperties(); + } catch (RuntimeException e) { + return Collections.emptyMap(); + } + } + + /** + * Every element of the Pipeline that has an Element Data key, with the + * members of a parallel group found through the available properties, + * whose metadata names the element that owns each property. + */ + private static List> elementsOf( + Pipeline pipeline, + Map> available) { + Map, Boolean> seen = new IdentityHashMap<>(); + List> elements = new ArrayList<>(); + for (Object top : pipeline.getFlowElements()) { + addElement((FlowElement) top, seen, elements); + } + for (Map byName : + available.values()) { + for (ElementPropertyMetaData property : byName.values()) { + FlowElement owner = property.getElement(); + if (owner != null) { + addElement(owner, seen, elements); + } + } + } + return elements; + } + + /** + * Add an element that has an Element Data key. A parallel group has + * none, and is passed over. + */ + private static void addElement( + FlowElement element, + Map, Boolean> seen, + List> elements) { + if (seen.containsKey(element)) { + return; + } + seen.put(element, Boolean.TRUE); + try { + element.getElementDataKey(); + } catch (UnsupportedOperationException e) { + return; + } + elements.add(element); + } + + private static boolean allLoaded(Pipeline pipeline) { + for (Object element : pipeline.getFlowElements()) { + if (((FlowElement) element).hasLoadedProperties() == false) { + return false; + } + } + return true; + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + + /** + * Write the build log for each script, being one information line + * giving the name, the version, the format, the source and the output + * property, the compiled model as canonical JSON at debug level, and a + * warning carrying the note of a deprecated script. + */ + void logScripts() { + for (CompiledScript script : scripts) { + DerivedScript model = script.getModel(); + logger.info("Derived property script '" + model.getName() + + "' version " + model.getVersion() + ", format " + + model.getFormat() + ", from " + model.getSource() + + ", produces '" + ELEMENT_DATA_KEY + "." + + model.getOutput().getName() + "'"); + if (logger.isDebugEnabled()) { + logger.debug("Derived property script '" + model.getName() + + "' compiled to:\n" + CanonicalJson.print(model)); + } + if (model.isDeprecated()) { + logger.warn("Derived property script '" + model.getName() + + "' is deprecated. " + model.getDeprecationNote()); + } + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementBuilder.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementBuilder.java new file mode 100644 index 000000000..18646f233 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementBuilder.java @@ -0,0 +1,345 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.annotations.ElementBuilder; +import fiftyone.pipeline.derivedproperty.BuiltInScript; +import fiftyone.pipeline.derivedproperty.DerivedScriptFault; +import fiftyone.pipeline.derivedproperty.DerivedScriptValidationException; +import fiftyone.pipeline.derivedproperty.data.DerivedPropertyData; +import fiftyone.pipeline.derivedproperty.data.IDerivedPropertyData; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryIteratorException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Builds a {@link DerivedPropertyElement} from scripts given in three ways, + * which can be mixed, being scripts shipped in this package, files on disk + * and text in code. There is no way to fetch a script over the network. + *

    + * Every script is parsed and validated when {@link #build()} is called, + * and every fault in every script is reported at once, in one + * {@link DerivedScriptValidationException}. + *

    + * A configuration file names this builder as + * {@code DerivedPropertyElement}, with {@code Scripts} naming built-in + * scripts and {@code ScriptFiles} naming files, each a comma separated + * list. Script text from code is not available from configuration. + *

    + * Every Derived Property Element in a Pipeline writes into the same Element + * Data, so two elements this builder makes must not be members of the same + * parallel group. + */ +@ElementBuilder +public class DerivedPropertyElementBuilder { + + /** + * The characters that make a file name a pattern. + */ + private static final String WILDCARDS = "*?[{"; + + /** + * A script as given, before it is validated. + */ + private static final class ScriptText { + private final String name; + private final String text; + private final String source; + + ScriptText(String name, String text, String source) { + this.name = name; + this.text = text; + this.source = source; + } + } + + private final ILoggerFactory loggerFactory; + private final List scripts = new ArrayList<>(); + private final List faults = new ArrayList<>(); + + /** + * Construct a new builder. + * @param loggerFactory the logger factory for the element and the + * Element Data it creates + */ + public DerivedPropertyElementBuilder(ILoggerFactory loggerFactory) { + if (loggerFactory == null) { + throw new IllegalArgumentException("loggerFactory"); + } + this.loggerFactory = loggerFactory; + } + + /** + * Add a script shipped in this package. + * @param script the script + * @return this builder + */ + public DerivedPropertyElementBuilder addScript(BuiltInScript script) { + String text = script.readText(); + if (text == null) { + faults.add(new DerivedScriptFault(script.name(), script.name(), + "", 0, "the built-in script '" + script.name() + "' is not " + + "in this package, which was built without the " + + "derived-properties scripts")); + return this; + } + scripts.add(new ScriptText(script.name(), text, script.name())); + return this; + } + + /** + * Add every file the path matches. The file name may hold the wildcards + * {@code *} and {@code ?}, as in {@code derived/*.yaml}. A script's Name + * must equal its file name without the extension. A path matching no + * file is a fault. + * @param path the path, relative to the working directory or absolute + * @return this builder + */ + public DerivedPropertyElementBuilder addScriptFile(String path) { + List matches = matches(path); + if (matches.isEmpty()) { + faults.add(new DerivedScriptFault("", path, "", 0, + "no file matches '" + path + "'")); + return this; + } + for (Path match : matches) { + String fileName = match.getFileName().toString(); + int dot = fileName.lastIndexOf('.'); + String name = dot > 0 ? fileName.substring(0, dot) : fileName; + try { + String text = new String( + Files.readAllBytes(match), StandardCharsets.UTF_8); + scripts.add(new ScriptText(name, text, match.toString())); + } catch (IOException e) { + faults.add(new DerivedScriptFault(name, match.toString(), "", + 0, "the file cannot be read: " + e.getMessage())); + } + } + return this; + } + + /** + * Add a script from text in code, read as YAML or JSON as the text + * shows. + * @param name the name the script must have + * @param content the text of the script + * @return this builder + */ + public DerivedPropertyElementBuilder addScript( + String name, + String content) { + scripts.add(new ScriptText(name, content, "code")); + return this; + } + + /** + * Add built-in scripts by name, which is how the {@code Scripts} build + * parameter of a configuration file reaches the builder. + * @param names the names, separated by commas + * @return this builder + */ + public DerivedPropertyElementBuilder setScripts(String names) { + for (String name : split(names)) { + BuiltInScript script = BuiltInScript.fromName(name); + if (script == null) { + faults.add(new DerivedScriptFault(name, name, "", 0, "'" + + name + "' is not a built-in script. The built-in " + + "scripts are " + BuiltInScript.names())); + } else { + addScript(script); + } + } + return this; + } + + /** + * Add script files by path, which is how the {@code ScriptFiles} build + * parameter of a configuration file reaches the builder. + * @param paths the paths, separated by commas, each of which may hold + * wildcards in its file name + * @return this builder + */ + public DerivedPropertyElementBuilder setScriptFiles(String paths) { + for (String path : split(paths)) { + addScriptFile(path); + } + return this; + } + + /** + * Validate every script and build the element. + * @return a new element + * @throws DerivedScriptValidationException listing every fault in every + * script, where any script has one + * @throws IllegalStateException where no script was added + */ + public DerivedPropertyElement build() { + if (scripts.isEmpty() && faults.isEmpty()) { + throw new IllegalStateException( + "A Derived Property Element needs at least one script. Add " + + "one with addScript or addScriptFile, or name one with the " + + "Scripts or ScriptFiles build parameter."); + } + List all = new ArrayList<>(faults); + List models = new ArrayList<>(); + for (ScriptText script : scripts) { + DerivedScriptValidator.Result result = DerivedScriptValidator + .validate(script.text, script.name, script.source); + all.addAll(result.getFaults()); + if (result.getScript() != null) { + models.add(result.getScript()); + } + } + all.addAll(duplicates(models)); + if (all.isEmpty() == false) { + throw new DerivedScriptValidationException(all); + } + Logger dataLogger = + loggerFactory.getLogger(DerivedPropertyData.class.getName()); + DerivedPropertyElement element = new DerivedPropertyElement( + loggerFactory.getLogger(DerivedPropertyElement.class.getName()), + (flowData, flowElement) -> + new DerivedPropertyData(dataLogger, flowData), + models); + element.logScripts(); + return element; + } + + /** + * Two scripts with the same name, or producing the same output + * property, as there is one script per name and one value per output + * property. + */ + private static List duplicates( + List models) { + List found = new ArrayList<>(); + Map names = new HashMap<>(); + Map outputs = new HashMap<>(); + for (DerivedScript model : models) { + String name = model.getName().toLowerCase(Locale.ROOT); + DerivedScript first = names.get(name); + if (first != null) { + found.add(new DerivedScriptFault(model.getName(), + model.getSource(), "Name", 0, "the script '" + + model.getName() + "' is added more than once, from " + + first.getSource() + " and from " + model.getSource())); + } else { + names.put(name, model); + } + String output = + model.getOutput().getName().toLowerCase(Locale.ROOT); + first = outputs.get(output); + if (first != null) { + found.add(new DerivedScriptFault(model.getName(), + model.getSource(), "Output.Name", 0, "the output " + + "property '" + model.getOutput().getName() + "' is " + + "also produced by the script '" + first.getName() + + "' from " + first.getSource())); + } else { + outputs.put(output, model); + } + } + return found; + } + + /** + * The entries of a comma separated list, with the white space around + * each removed and empty entries left out. + */ + private static List split(String list) { + List entries = new ArrayList<>(); + if (list == null) { + return entries; + } + for (String entry : list.split(",")) { + String trimmed = entry.trim(); + if (trimmed.isEmpty() == false) { + entries.add(trimmed); + } + } + return entries; + } + + /** + * The files a path matches, in name order. Only the file name may hold + * wildcards, which are read as a glob. + */ + private static List matches(String pattern) { + List found = new ArrayList<>(); + if (pattern == null || pattern.isEmpty()) { + return found; + } + File file = new File(pattern); + String glob = file.getName(); + try { + if (hasWildcard(glob) == false) { + Path path = file.toPath(); + if (Files.isRegularFile(path)) { + found.add(path); + } + return found; + } + File parent = file.getParentFile(); + Path directory = parent == null ? + new File(".").toPath() : parent.toPath(); + try (DirectoryStream stream = + Files.newDirectoryStream(directory, glob)) { + for (Path match : stream) { + if (Files.isRegularFile(match)) { + found.add(parent == null ? + match.getFileName() : match); + } + } + } + } catch (IOException | DirectoryIteratorException | + IllegalArgumentException e) { + // An invalid path or pattern matches no file. + return Collections.emptyList(); + } + found.sort(Comparator.comparing(Path::toString)); + return found; + } + + private static boolean hasWildcard(String name) { + for (int i = 0; i < name.length(); i++) { + if (WILDCARDS.indexOf(name.charAt(i)) >= 0) { + return true; + } + } + return false; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyMetaData.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyMetaData.java new file mode 100644 index 000000000..614774c7d --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyMetaData.java @@ -0,0 +1,97 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +/** + * The full Property metadata block of one script a Derived Property Element + * holds. Its {@link #getOutput()} is the whole definition of the derived + * property. + */ +public final class DerivedPropertyMetaData { + + private final String name; + private final String version; + private final int format; + private final boolean deprecated; + private final String deprecationNote; + private final String source; + private final DerivedOutputMetaData output; + + DerivedPropertyMetaData(DerivedScript script) { + this.name = script.getName(); + this.version = script.getVersion(); + this.format = script.getFormat(); + this.deprecated = script.isDeprecated(); + this.deprecationNote = script.getDeprecationNote(); + this.source = script.getSource(); + this.output = new DerivedOutputMetaData(script.getOutput()); + } + + /** + * @return the script's name, which is what selects it + */ + public String getName() { + return name; + } + + /** + * @return the author's semantic version, which plays no part in + * selecting the script + */ + public String getVersion() { + return version; + } + + /** + * @return the version of the script language, which is 1 + */ + public int getFormat() { + return format; + } + + public boolean isDeprecated() { + return deprecated; + } + + /** + * @return what to use instead, or empty where the script is not + * deprecated + */ + public String getDeprecationNote() { + return deprecationNote; + } + + /** + * @return the built-in script's name, the file path, or "code" + */ + public String getSource() { + return source; + } + + /** + * @return the definition of the property the script produces + */ + public DerivedOutputMetaData getOutput() { + return output; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScript.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScript.java new file mode 100644 index 000000000..6c253198c --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScript.java @@ -0,0 +1,185 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * A validated script, the model every language builds from the same text. + * Nothing in it changes once it is built. + */ +final class DerivedScript { + + /** + * A check, which is a named condition. + */ + static final class NamedCheck { + private final String name; + private final Condition condition; + + NamedCheck(String name, Condition condition) { + this.name = name; + this.condition = condition; + } + + String getName() { + return name; + } + + Condition getCondition() { + return condition; + } + } + + /** + * A When with a Then, or, where the condition is null, an Else. + */ + static final class Rule { + private final Condition when; + private final Literal value; + + Rule(Condition when, Literal value) { + this.when = when; + this.value = value; + } + + /** + * @return the condition, or null for an Else + */ + Condition getWhen() { + return when; + } + + Literal getValue() { + return value; + } + } + + private final int format; + private final String name; + private final String version; + private final boolean deprecated; + private final String deprecationNote; + private final String source; + private final OutputModel output; + private final List properties; + private final List checks; + private final List rules; + + /** + * @param properties the source properties in the order the script first + * names them, checks before rules. A comparison reads + * its property from the slot with the same index. + */ + DerivedScript( + int format, + String name, + String version, + boolean deprecated, + String deprecationNote, + String source, + OutputModel output, + List properties, + List checks, + List rules) { + this.format = format; + this.name = name; + this.version = version; + this.deprecated = deprecated; + this.deprecationNote = deprecationNote; + this.source = source; + this.output = output; + this.properties = Collections.unmodifiableList( + new ArrayList<>(properties)); + this.checks = Collections.unmodifiableList(new ArrayList<>(checks)); + this.rules = Collections.unmodifiableList(new ArrayList<>(rules)); + } + + int getFormat() { + return format; + } + + String getName() { + return name; + } + + String getVersion() { + return version; + } + + boolean isDeprecated() { + return deprecated; + } + + String getDeprecationNote() { + return deprecationNote; + } + + /** + * @return the built-in script's name, the file path, or "code" + */ + String getSource() { + return source; + } + + OutputModel getOutput() { + return output; + } + + List getProperties() { + return properties; + } + + List getChecks() { + return checks; + } + + List getRules() { + return rules; + } + + /** + * Run the script on the source properties read for it. Every slot is + * available, so every condition is true or false, and the Else every + * script ends in always matches. + * @param slots the source properties, in the order of + * {@link #getProperties()} + * @return the value of the first rule whose condition is true + */ + Literal evaluate(Slot[] slots) { + boolean[] results = new boolean[checks.size()]; + for (int i = 0; i < results.length; i++) { + results[i] = checks.get(i).getCondition().test(slots, results); + } + for (Rule rule : rules) { + if (rule.getWhen() == null || + rule.getWhen().test(slots, results)) { + return rule.getValue(); + } + } + throw new IllegalStateException( + "The rules of '" + output.getName() + "' do not end in an Else."); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptParser.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptParser.java new file mode 100644 index 000000000..188b1066a --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptParser.java @@ -0,0 +1,286 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import org.snakeyaml.engine.v2.api.LoadSettings; +import org.snakeyaml.engine.v2.api.lowlevel.Compose; +import org.snakeyaml.engine.v2.exceptions.Mark; +import org.snakeyaml.engine.v2.exceptions.MarkedYamlEngineException; +import org.snakeyaml.engine.v2.exceptions.YamlEngineException; +import org.snakeyaml.engine.v2.nodes.MappingNode; +import org.snakeyaml.engine.v2.nodes.Node; +import org.snakeyaml.engine.v2.nodes.NodeTuple; +import org.snakeyaml.engine.v2.nodes.ScalarNode; +import org.snakeyaml.engine.v2.nodes.SequenceNode; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Pattern; + +/** + * Reads the text of a script, as YAML or as JSON, into {@link ScriptNode}s. + *

    + * JSON is read as the JSON subset of YAML 1.2 by the same reader. The + * nodes are composed rather than constructed, so every node keeps its line, + * and a plain scalar is resolved here by the YAML 1.2 core schema, under + * which only {@code true} and {@code false} in their three spellings are + * booleans, so {@code yes} and {@code NO} stay strings. A quoted scalar is + * always a string. + *

    + * See + * YAML 1.2.2, + * 10.3.2 Tag Resolution and + * format + * 1, two places where the languages are allowed to differ. + */ +final class DerivedScriptParser { + + /** + * How deep a value may nest before the script is refused, which also + * stops an alias that refers to itself. + */ + private static final int MAX_DEPTH = 100; + + private static final Pattern DECIMAL_INT = Pattern.compile("[-+]?[0-9]+"); + private static final Pattern OCTAL_INT = Pattern.compile("0o[0-7]+"); + private static final Pattern HEX_INT = Pattern.compile("0x[0-9a-fA-F]+"); + private static final Pattern FLOAT = Pattern.compile( + "[-+]?(\\.[0-9]+|[0-9]+(\\.[0-9]*)?)([eE][-+]?[0-9]+)?"); + private static final Pattern INFINITY = + Pattern.compile("[-+]?\\.(inf|Inf|INF)"); + private static final Pattern NOT_A_NUMBER = + Pattern.compile("\\.(nan|NaN|NAN)"); + + private static final int OCTAL_RADIX = 8; + private static final int HEX_RADIX = 16; + + /** + * The length of the 0o or 0x prefix. + */ + private static final int RADIX_PREFIX_LENGTH = 2; + + private DerivedScriptParser() { + } + + /** + * Parse the text of a script. + * @param text the script, as YAML or JSON + * @return the document, which is always a mapping + * @throws ScriptParseException where the text is not YAML or JSON, is + * empty, is not a mapping, or writes a key twice + */ + static ScriptNode parse(String text) throws ScriptParseException { + if (text == null) { + throw new ScriptParseException("the script is empty", 0); + } + String what = startsLikeJson(text) ? "JSON" : "YAML"; + Node root; + try { + root = new Compose(LoadSettings.builder().build()) + .composeString(text) + .orElse(null); + } catch (MarkedYamlEngineException e) { + Optional mark = e.getProblemMark().isPresent() ? + e.getProblemMark() : e.getContextMark(); + String problem = e.getProblem() != null ? + e.getProblem() : e.getMessage(); + throw new ScriptParseException( + "the text is not valid " + what + ": " + problem, + mark.map(m -> m.getLine() + 1).orElse(0)); + } catch (YamlEngineException e) { + throw new ScriptParseException( + "the text is not valid " + what + ": " + e.getMessage(), 0); + } + if (root == null) { + throw new ScriptParseException("the script is empty", 0); + } + ScriptNode document = convert(root, 0); + if (document.getKind() == ScriptNode.Kind.NULL) { + throw new ScriptParseException("the script is empty", 0); + } + if (document.getKind() != ScriptNode.Kind.MAP) { + throw new ScriptParseException( + "the script must be a mapping of keys to values at the top " + + "level", + 0); + } + requireDistinctKeys(document); + return document; + } + + /** + * Whether the text opens with a brace once leading white space is + * passed over, which is how the reference tools tell JSON from YAML. + */ + private static boolean startsLikeJson(String text) { + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + if (JsText.isWhitespace(c) == false) { + return c == '{'; + } + } + return false; + } + + private static ScriptNode convert(Node node, int depth) + throws ScriptParseException { + int line = node.getStartMark().map(m -> m.getLine() + 1).orElse(0); + if (depth > MAX_DEPTH) { + throw new ScriptParseException( + "the text is not valid YAML: values nest too deeply", line); + } + if (node instanceof MappingNode) { + List keys = new ArrayList<>(); + List values = new ArrayList<>(); + for (NodeTuple tuple : ((MappingNode) node).getValue()) { + keys.add(keyText(tuple.getKeyNode())); + values.add(convert(tuple.getValueNode(), depth + 1)); + } + return ScriptNode.ofMap(keys, values, line); + } + if (node instanceof SequenceNode) { + List items = new ArrayList<>(); + for (Node item : ((SequenceNode) node).getValue()) { + items.add(convert(item, depth + 1)); + } + return ScriptNode.ofList(items, line); + } + if (node instanceof ScalarNode) { + return scalar((ScalarNode) node, line); + } + return ScriptNode.ofNull(line); + } + + /** + * A key is the text it is written as, as JavaScript makes every key of + * an object a string. + */ + private static String keyText(Node key) { + return key instanceof ScalarNode ? ((ScalarNode) key).getValue() : ""; + } + + /** + * Resolve a scalar. A plain scalar takes the tag the YAML 1.2 core + * schema gives it, and any other scalar is a string. + */ + static ScriptNode scalar(ScalarNode node, int line) { + String value = node.getValue(); + if (node.isPlain() == false) { + return ScriptNode.ofString(value, line); + } + switch (value) { + case "": + case "~": + case "null": + case "Null": + case "NULL": + return ScriptNode.ofNull(line); + case "true": + case "True": + case "TRUE": + return ScriptNode.ofBool(true, line); + case "false": + case "False": + case "FALSE": + return ScriptNode.ofBool(false, line); + default: + break; + } + Double number = coreNumber(value); + return number == null ? + ScriptNode.ofString(value, line) : + ScriptNode.ofNumber(number, line); + } + + /** + * The number a plain scalar is under the core schema, or null where it + * is not one. + */ + private static Double coreNumber(String value) { + if (DECIMAL_INT.matcher(value).matches() || + FLOAT.matcher(value).matches()) { + return Double.parseDouble(value); + } + if (OCTAL_INT.matcher(value).matches()) { + return new BigInteger( + value.substring(RADIX_PREFIX_LENGTH), OCTAL_RADIX) + .doubleValue(); + } + if (HEX_INT.matcher(value).matches()) { + return new BigInteger( + value.substring(RADIX_PREFIX_LENGTH), HEX_RADIX) + .doubleValue(); + } + if (INFINITY.matcher(value).matches()) { + return value.charAt(0) == '-' ? + Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY; + } + if (NOT_A_NUMBER.matcher(value).matches()) { + return Double.NaN; + } + return null; + } + + /** + * Refuse two keys of one mapping that differ only in case, because keys + * are matched without regard to case and one of the two would be + * dropped, and refuse a key written twice for the same reason. + */ + private static void requireDistinctKeys(ScriptNode node) + throws ScriptParseException { + if (node.getKind() == ScriptNode.Kind.LIST) { + for (ScriptNode item : node.getItems()) { + requireDistinctKeys(item); + } + return; + } + if (node.getKind() != ScriptNode.Kind.MAP) { + return; + } + Map seen = new HashMap<>(); + for (int i = 0; i < node.getKeys().size(); i++) { + String key = node.getKeys().get(i); + String first = seen.get(key.toLowerCase(Locale.ROOT)); + if (first != null) { + if (first.equals(key)) { + throw new ScriptParseException( + "the key '" + key + "' is written twice in one " + + "mapping, so one of the two would be dropped", + node.getLine()); + } + throw new ScriptParseException( + "the keys '" + first + "' and '" + key + "' differ only " + + "in case, and keys are matched without regard to " + + "case, so one of the two would be dropped", + node.getLine()); + } + seen.put(key.toLowerCase(Locale.ROOT), key); + requireDistinctKeys(node.getValues().get(i)); + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptValidator.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptValidator.java new file mode 100644 index 000000000..a801ba562 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptValidator.java @@ -0,0 +1,1168 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.derivedproperty.DerivedScriptFault; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * Checks a parsed script and builds its model, collecting every fault + * rather than stopping at the first. The paths and the messages are those + * of the reference implementation, tools/validate.mjs in the + * derived-properties repository, so a fault reads the same in every + * language. + *

    + * See format + * 1. + */ +final class DerivedScriptValidator { + + /** + * The result of validating one script. + */ + static final class Result { + private final DerivedScript script; + private final List faults; + + Result(DerivedScript script, List faults) { + this.script = script; + this.faults = Collections.unmodifiableList(faults); + } + + /** + * @return the model, or null where the script has a fault + */ + DerivedScript getScript() { + return script; + } + + List getFaults() { + return faults; + } + } + + static final List VALUE_TYPES = + Arrays.asList("string", "bool", "int", "double"); + static final List AGGREGATE_NAMES = + Arrays.asList("Passed", "Failed"); + static final List TOP_LEVEL_KEYS = Arrays.asList( + "Format", "Name", "Version", "Deprecated", "DeprecationNote", + "Output", "Checks", "Rules"); + /** + * The keys of Output, in the order the canonical form prints them. + */ + static final List OUTPUT_KEYS = Arrays.asList( + "Name", "Description", "ValueType", "StoredValueType", + "DefaultValue", "IsList", "IsMandatory", "IsObsolete", "Category", + "IsPopular", "ExportValues", "Url", "DisplayOrder", "PropertyId", + "VendorIds", "Dependencies", "Values"); + private static final List COUNT_OPERATORS = + Arrays.asList("Eq", "Ne", "Gt", "Ge", "Lt", "Le"); + + private static final Pattern IDENTIFIER = + Pattern.compile("^[A-Za-z][A-Za-z0-9]*$"); + private static final Pattern SOURCE_PATTERN = + Pattern.compile("^[A-Za-z][A-Za-z0-9]*\\.[A-Za-z][A-Za-z0-9]*$"); + private static final Pattern SEMANTIC_VERSION = Pattern.compile( + "^\\d+\\.\\d+\\.\\d+(?:-[0-9A-Za-z.-]+(?:\\.[0-9A-Za-z.-]+)*)?" + + "(?:\\+[0-9A-Za-z.-]+)?$"); + + /** + * Where a source property was first seen and the type inferred there, + * so a conflict can name both places. + */ + private static final class PropertyUse { + private final int slot; + private final ValueType type; + private final String typePath; + + PropertyUse(int slot, ValueType type, String typePath) { + this.slot = slot; + this.type = type; + this.typePath = typePath; + } + } + + private final ScriptNode document; + private final String script; + private final String source; + private final List faults = new ArrayList<>(); + private final Map uses = new HashMap<>(); + private final List properties = new ArrayList<>(); + + private DerivedScriptValidator( + ScriptNode document, + String script, + String source) { + this.document = document; + this.script = script; + this.source = source; + } + + /** + * Parse and validate the text of a script. + * @param text the script, as YAML or JSON + * @param name the file name without its extension, which the script's + * Name must equal, or empty where there is none + * @param source the built-in script's name, the file path, or "code" + * @return the model, or every fault found + */ + static Result validate(String text, String name, String source) { + String expected = name == null ? "" : name; + ScriptNode document; + try { + document = DerivedScriptParser.parse(text); + } catch (ScriptParseException e) { + List faults = new ArrayList<>(); + faults.add(new DerivedScriptFault( + expected, source, "", e.getLine(), e.getMessage())); + return new Result(null, faults); + } + DerivedScriptValidator validator = + new DerivedScriptValidator(document, expected, source); + DerivedScript model = validator.buildModel(); + if (validator.faults.isEmpty() == false) { + return new Result(null, validator.faults); + } + return new Result(model, validator.faults); + } + + private void fault(String path, ScriptNode at, String message) { + int line = at != null && at.isCollection() ? at.getLine() : 0; + faults.add(new DerivedScriptFault( + script, source, path, line, message)); + } + + private DerivedScript buildModel() { + for (String key : unknownKeys(document, TOP_LEVEL_KEYS)) { + fault(key, document, "unknown key '" + key + "' at the top " + + "level. Expected one of " + String.join(", ", TOP_LEVEL_KEYS)); + } + int format = readFormat(); + String name = readName(); + String version = readVersion(); + boolean deprecated = readDeprecated(); + String deprecationNote = readDeprecationNote(deprecated); + OutputModel.Draft output = readOutput(); + // Checks are read before rules, so that a rule can refer to one. + List checkNames = readCheckNames(); + List checks = readChecks(checkNames); + List rules = readRules(output, checkNames); + List computed = new ArrayList<>(); + if (output.dependencies == null) { + for (SourceProperty property : properties) { + computed.add(property.getName()); + } + } + return new DerivedScript(format, name, version, deprecated, + deprecationNote, source, new OutputModel(output, computed), + properties, checks, rules); + } + + private int readFormat() { + ScriptNode value = document.get("Format"); + if (value == null) { + fault("Format", document, "required key 'Format' is missing"); + return 0; + } + if (value.getKind() != ScriptNode.Kind.NUMBER || + value.getNumber() != 1) { + fault("Format", document, + "Format must be 1, found " + jsonOf(value)); + return 0; + } + return 1; + } + + private String readName() { + ScriptNode value = document.get("Name"); + if (value == null) { + fault("Name", document, "required key 'Name' is missing"); + return ""; + } + if (value.getKind() != ScriptNode.Kind.STRING) { + fault("Name", document, + "Name expected a string, found " + typeOf(value)); + return ""; + } + if (IDENTIFIER.matcher(value.getText()).matches() == false) { + fault("Name", document, "script name '" + value.getText() + + "' does not match the pattern " + IDENTIFIER.pattern()); + return value.getText(); + } + if (script.isEmpty() == false && + script.equals(value.getText()) == false) { + fault("Name", document, "script name '" + value.getText() + + "' must equal the file name '" + script + "'"); + } + return value.getText(); + } + + private String readVersion() { + ScriptNode value = document.get("Version"); + if (value == null) { + fault("Version", document, "required key 'Version' is missing"); + return ""; + } + if (value.getKind() != ScriptNode.Kind.STRING || + SEMANTIC_VERSION.matcher(value.getText()).matches() == false) { + fault("Version", document, "Version expected a semantic " + + "version such as 1.0.0, found " + jsonOf(value)); + return ""; + } + return value.getText(); + } + + private boolean readDeprecated() { + ScriptNode deprecated = document.get("Deprecated"); + if (deprecated == null) { + return false; + } + if (deprecated.getKind() != ScriptNode.Kind.BOOL) { + fault("Deprecated", document, + "Deprecated expected a boolean, found " + typeOf(deprecated)); + return false; + } + return deprecated.getBool(); + } + + private String readDeprecationNote(boolean deprecated) { + ScriptNode note = document.get("DeprecationNote"); + String text = ""; + if (note != null && note.getKind() != ScriptNode.Kind.STRING) { + fault("DeprecationNote", document, + "DeprecationNote expected a string, found " + typeOf(note)); + } else if (note != null) { + text = note.getText(); + } + if (deprecated && text.isEmpty()) { + fault("DeprecationNote", document, "a deprecated script must " + + "say what to use instead in DeprecationNote"); + } + if (deprecated == false && text.isEmpty() == false) { + fault("DeprecationNote", document, + "DeprecationNote is only allowed when Deprecated is true"); + } + return text; + } + + private OutputModel.Draft readOutput() { + OutputModel.Draft output = new OutputModel.Draft(); + String fallbackName = script.isEmpty() ? "unknown" : script; + ScriptNode raw = document.get("Output"); + if (raw == null) { + fault("Output", document, "required key 'Output' is missing"); + output.name = fallbackName; + return output; + } + if (raw.getKind() != ScriptNode.Kind.MAP) { + fault("Output", document, + "Output expected a mapping, found " + typeOf(raw)); + output.name = fallbackName; + return output; + } + for (String key : unknownKeys(raw, OUTPUT_KEYS)) { + fault("Output." + key, raw, "unknown key '" + key + "' under " + + "Output. A typo in a metadata field is a fault rather than " + + "a value that is quietly dropped"); + } + + ScriptNode name = raw.get("Name"); + if (name == null) { + fault("Output.Name", raw, "required key 'Name' is missing"); + } else if (name.getKind() != ScriptNode.Kind.STRING || + IDENTIFIER.matcher(name.getText()).matches() == false) { + fault("Output.Name", raw, "Output.Name " + jsonOf(name) + + " does not match the pattern " + IDENTIFIER.pattern()); + } else { + output.name = name.getText(); + } + if (output.name.isEmpty()) { + output.name = fallbackName; + } + + ScriptNode description = raw.get("Description"); + if (description == null) { + fault("Output.Description", raw, "required key 'Description' " + + "is missing. Say what the property asserts, not how far to " + + "trust it"); + } else if (description.getKind() != ScriptNode.Kind.STRING || + JsText.trim(description.getText()).isEmpty()) { + fault("Output.Description", raw, "Output.Description expected " + + "a non empty string, found " + typeOf(description)); + } else { + output.description = description.getText(); + } + + ScriptNode valueType = raw.get("ValueType"); + if (valueType == null) { + fault("Output.ValueType", raw, + "required key 'ValueType' is missing"); + } else if (valueType.getKind() != ScriptNode.Kind.STRING || + ValueType.fromText(valueType.getText()) == null) { + fault("Output.ValueType", raw, "Output.ValueType '" + + displayOf(valueType) + "' is not allowed in format 1. " + + "Expected one of " + String.join(", ", VALUE_TYPES)); + } else { + output.valueType = ValueType.fromText(valueType.getText()); + } + + ScriptNode isList = raw.get("IsList"); + if (isList == null) { + fault("Output.IsList", raw, "required key 'IsList' is missing"); + } else if (isList.getKind() != ScriptNode.Kind.BOOL) { + fault("Output.IsList", raw, + "Output.IsList expected a boolean, found " + typeOf(isList)); + } else if (isList.getBool()) { + fault("Output.IsList", raw, "Output.IsList must be false in " + + "format 1. List outputs are deferred"); + } + + readValues(raw, output); + + output.isMandatory = readBoolean(raw, "IsMandatory"); + output.isObsolete = readBoolean(raw, "IsObsolete"); + output.isPopular = readBoolean(raw, "IsPopular"); + output.exportValues = readBoolean(raw, "ExportValues"); + output.category = readString(raw, "Category"); + output.url = readString(raw, "Url"); + output.storedValueType = readString(raw, "StoredValueType"); + output.displayOrder = readInteger(raw, "DisplayOrder"); + output.propertyId = readInteger(raw, "PropertyId"); + output.vendorIds = readList(raw, "VendorIds"); + output.dependencies = readList(raw, "Dependencies"); + + // DefaultValue is the string form of the value, so it is checked + // against the value type and the value list. + ScriptNode defaultValue = raw.get("DefaultValue"); + if (defaultValue != null) { + if (defaultValue.getKind() != ScriptNode.Kind.STRING) { + fault("Output.DefaultValue", raw, "Output.DefaultValue " + + "expected a string holding the string form of the " + + "value, found " + typeOf(defaultValue)); + } else { + String text = defaultValue.getText(); + output.defaultValue = text; + if (output.valueType != null && + SourceValues.convertString(text, output.valueType) == + null) { + fault("Output.DefaultValue", raw, "Output.DefaultValue '" + + text + "' cannot be read as " + output.valueType); + } + if (output.hasValues && output.listsValue(text) == false) { + fault("Output.DefaultValue", raw, "Output.DefaultValue '" + + text + "' is not one of the values listed under " + + "Output.Values"); + } + } + } + return output; + } + + private Boolean readBoolean(ScriptNode raw, String key) { + ScriptNode value = raw.get(key); + if (value == null) { + return null; + } + if (value.getKind() != ScriptNode.Kind.BOOL) { + fault("Output." + key, raw, "Output." + key + + " expected a boolean, found " + typeOf(value)); + return null; + } + return value.getBool(); + } + + private String readString(ScriptNode raw, String key) { + ScriptNode value = raw.get(key); + if (value == null) { + return null; + } + if (value.getKind() != ScriptNode.Kind.STRING) { + fault("Output." + key, raw, "Output." + key + + " expected a string, found " + typeOf(value)); + return null; + } + return value.getText(); + } + + private Long readInteger(ScriptNode raw, String key) { + ScriptNode value = raw.get(key); + if (value == null) { + return null; + } + if (value.getKind() != ScriptNode.Kind.NUMBER || + isWhole(value.getNumber()) == false) { + fault("Output." + key, raw, "Output." + key + + " expected an integer, found " + typeOf(value)); + return null; + } + return (long) value.getNumber(); + } + + private ScriptNode readList(ScriptNode raw, String key) { + ScriptNode value = raw.get(key); + if (value == null) { + return null; + } + if (value.getKind() != ScriptNode.Kind.LIST) { + fault("Output." + key, raw, "Output." + key + + " expected a list, found " + typeOf(value)); + return null; + } + return value; + } + + private void readValues(ScriptNode raw, OutputModel.Draft output) { + ScriptNode values = raw.get("Values"); + if (values == null) { + return; + } + if (values.getKind() != ScriptNode.Kind.LIST) { + fault("Output.Values", raw, + "Output.Values expected a list, found " + typeOf(values)); + return; + } + if (output.valueType != null && + output.valueType != ValueType.STRING && + output.valueType != ValueType.INT) { + fault("Output.Values", raw, "Output.Values is only allowed " + + "where ValueType is string or int, not " + output.valueType); + return; + } + for (int index = 0; index < values.getItems().size(); index++) { + ScriptNode entry = values.getItems().get(index); + String path = "Output.Values[" + index + "]"; + if (entry.getKind() != ScriptNode.Kind.MAP) { + fault(path, values, "a value expected a mapping of Name " + + "and Description, found " + typeOf(entry)); + continue; + } + for (String key : unknownKeys( + entry, Arrays.asList("Name", "Description"))) { + fault(path + "." + key, entry, "unknown key '" + key + + "' in a value. Expected Name and Description"); + } + ScriptNode name = entry.get("Name"); + if (name == null || (name.getKind() != ScriptNode.Kind.STRING && + name.getKind() != ScriptNode.Kind.NUMBER)) { + fault(path + ".Name", entry, "a value must have a Name"); + continue; + } + String text = null; + ScriptNode description = entry.get("Description"); + if (description != null) { + if (description.getKind() != ScriptNode.Kind.STRING) { + fault(path + ".Description", entry, "a value " + + "Description expected a string, found " + + typeOf(description)); + } + text = description.getText(); + } + output.values.add( + new OutputModel.ValueEntry(Literal.of(name), text)); + } + List seen = new ArrayList<>(); + for (OutputModel.ValueEntry entry : output.values) { + String key = entry.getName().toString(); + if (seen.contains(key)) { + fault("Output.Values", values, + "the value '" + key + "' is listed more than once"); + } + seen.add(key); + } + output.hasValues = true; + } + + private List readCheckNames() { + ScriptNode raw = document.get("Checks"); + if (raw == null) { + return Collections.emptyList(); + } + if (raw.getKind() != ScriptNode.Kind.MAP) { + fault("Checks", document, "Checks expected a mapping of names " + + "to conditions, found " + typeOf(raw)); + return Collections.emptyList(); + } + List names = new ArrayList<>(); + for (String name : raw.getKeys()) { + if (IDENTIFIER.matcher(name).matches() == false) { + fault("Checks." + name, raw, "check name '" + name + + "' does not match the pattern " + IDENTIFIER.pattern()); + continue; + } + names.add(name); + } + return names; + } + + private List readChecks( + List checkNames) { + List checks = new ArrayList<>(); + ScriptNode raw = document.get("Checks"); + if (raw == null || raw.getKind() != ScriptNode.Kind.MAP) { + return checks; + } + for (String name : checkNames) { + Condition condition = readCondition( + raw.valueOf(name), "Checks." + name, raw, checkNames); + checks.add(new DerivedScript.NamedCheck(name, condition)); + } + return checks; + } + + private List readRules( + OutputModel.Draft output, + List checkNames) { + List rules = new ArrayList<>(); + ScriptNode raw = document.get("Rules"); + if (raw == null) { + fault("Rules", document, "required key 'Rules' is missing"); + return rules; + } + if (raw.getKind() != ScriptNode.Kind.LIST) { + fault("Rules", document, + "Rules expected a list, found " + typeOf(raw)); + return rules; + } + List items = raw.getItems(); + if (items.isEmpty()) { + fault("Rules", raw, "Rules must hold at least one rule"); + return rules; + } + // Every script ends in an Else, so a script always chooses a value + // once its source properties have been read. + if (items.get(items.size() - 1).get("Else") == null) { + fault("Rules", raw, "the last rule must be an Else, which is " + + "what a script falls back to when no earlier rule matched"); + } + for (int index = 0; index < items.size(); index++) { + ScriptNode entry = items.get(index); + String path = "Rules[" + index + "]"; + if (entry.getKind() != ScriptNode.Kind.MAP) { + fault(path, raw, "a rule expected a mapping, found " + + typeOf(entry)); + continue; + } + for (String key : unknownKeys( + entry, Arrays.asList("When", "Then", "Else"))) { + fault(path + "." + key, entry, "unknown key '" + key + + "' in a rule. Expected When and Then, or Else"); + } + ScriptNode when = entry.get("When"); + ScriptNode then = entry.get("Then"); + ScriptNode otherwise = entry.get("Else"); + boolean isLast = index == items.size() - 1; + if (otherwise != null && when != null) { + fault(path, entry, "a rule has both When and Else. A rule " + + "is either When with Then, or Else on its own"); + continue; + } + if (otherwise != null && isLast == false) { + fault(path, entry, "Else is only allowed on the last rule"); + continue; + } + if (otherwise == null && when == null) { + fault(path, entry, + "a rule needs a When, or an Else on the last rule"); + continue; + } + if (when != null && then == null) { + fault(path, entry, "a rule with When needs a Then"); + continue; + } + Literal value = otherwise != null ? + readRuleValue(output, otherwise, path + ".Else", entry) : + readRuleValue(output, then, path + ".Then", entry); + Condition condition = when == null ? null : + readCondition(when, path + ".When", entry, checkNames); + rules.add(new DerivedScript.Rule(condition, value)); + } + return rules; + } + + /** + * Read a Then or an Else, which is a literal of Output.ValueType and + * one of Output.Values where that list is given. + */ + private Literal readRuleValue( + OutputModel.Draft output, + ScriptNode raw, + String path, + ScriptNode at) { + if (raw.getKind() == ScriptNode.Kind.NULL) { + fault(path, at, "a rule value is a null literal, which format 1 " + + "does not allow"); + return Literal.NONE; + } + if (raw.isCollection()) { + fault(path, at, "a rule value is a literal of the output value " + + "type, found " + typeOf(raw)); + return Literal.NONE; + } + if (output.valueType != null && + literalMatches(inferType(raw), output.valueType) == false) { + fault(path, at, "expected a " + output.valueType + + " to match Output.ValueType, found " + typeOf(raw)); + return Literal.NONE; + } + Literal value = Literal.of(raw); + if (output.hasValues && output.listsValue(value.toString()) == false) { + List names = new ArrayList<>(); + for (OutputModel.ValueEntry entry : output.values) { + names.add(entry.getName().toString()); + } + fault(path, at, "'" + value + "' is not one of the values " + + "listed under Output.Values (" + String.join(", ", names) + + ")"); + return Literal.NONE; + } + return value; + } + + private Condition readCondition( + ScriptNode raw, + String path, + ScriptNode parent, + List checkNames) { + if (raw == null || raw.getKind() != ScriptNode.Kind.MAP) { + fault(path, parent, + "a condition expected a mapping, found " + typeOf(raw)); + return Condition.faulted(); + } + if (raw.getKeys().isEmpty()) { + fault(path, raw, "a condition is empty"); + return Condition.faulted(); + } + if (raw.get("Property") != null) { + return readComparison(raw, path); + } + if (raw.get("Check") != null) { + return readCheckReference(raw, path, checkNames); + } + if (raw.get("Passed") != null || raw.get("Failed") != null) { + return readAggregate(raw, path, checkNames); + } + if (raw.get("All") != null || raw.get("Any") != null) { + String which = raw.get("All") != null ? "All" : "Any"; + if (raw.getKeys().size() != 1) { + fault(path, raw, which + " must be the only key of its " + + "condition, found " + String.join(", ", raw.getKeys())); + } + ScriptNode items = raw.get(which); + if (items.getKind() != ScriptNode.Kind.LIST) { + fault(path + "." + which, raw, which + " expected a list of " + + "conditions, found " + typeOf(items)); + return Condition.faulted(); + } + if (items.getItems().isEmpty()) { + fault(path + "." + which, raw, + which + " must list at least one condition"); + return Condition.faulted(); + } + Condition[] members = new Condition[items.getItems().size()]; + for (int index = 0; index < members.length; index++) { + members[index] = readCondition( + items.getItems().get(index), + path + "." + which + "[" + index + "]", + items, + checkNames); + } + return Condition.compound( + which.equals("All") ? + Condition.Kind.ALL : Condition.Kind.ANY, + members); + } + if (raw.get("Not") != null) { + if (raw.getKeys().size() != 1) { + fault(path, raw, "Not must be the only key of its " + + "condition, found " + String.join(", ", raw.getKeys())); + } + Condition item = readCondition( + raw.get("Not"), path + ".Not", raw, checkNames); + return Condition.compound( + Condition.Kind.NOT, new Condition[]{item}); + } + fault(path, raw, "a condition must be a comparison, a Check " + + "reference, an aggregate, All, Any or Not. Found the keys " + + String.join(", ", raw.getKeys())); + return Condition.faulted(); + } + + private Condition readComparison(ScriptNode raw, String path) { + String propertyKey = null; + List operatorKeys = new ArrayList<>(); + for (String key : raw.getKeys()) { + if (propertyKey == null && key.equalsIgnoreCase("Property")) { + propertyKey = key; + continue; + } + operatorKeys.add(key); + } + ScriptNode property = raw.valueOf(propertyKey); + if (property.getKind() != ScriptNode.Kind.STRING || + SOURCE_PATTERN.matcher(property.getText()).matches() == false) { + fault(path + ".Property", raw, "'" + displayOf(property) + + "' is not a source property. Write it as " + + "elementKey.PropertyName, for example device.IsCrawler"); + return Condition.faulted(); + } + if (operatorKeys.isEmpty()) { + fault(path, raw, "a comparison on '" + property.getText() + + "' has no operator. Expected exactly one of " + + Operator.names()); + return Condition.faulted(); + } + List known = new ArrayList<>(); + for (String key : operatorKeys) { + if (Operator.fromText(key) != null) { + known.add(key); + } else { + fault(path, raw, "unknown operator '" + key + + "', expected one of " + Operator.names()); + } + } + if (known.isEmpty()) { + return Condition.faulted(); + } + if (operatorKeys.size() > 1) { + fault(path, raw, "a condition takes exactly one operator, found " + + String.join(", ", operatorKeys)); + return Condition.faulted(); + } + Operator operator = Operator.fromText(known.get(0)); + ScriptNode operand = raw.valueOf(known.get(0)); + if (operand.getKind() == ScriptNode.Kind.NULL) { + fault(path + "." + operator, raw, "a null literal is not " + + "allowed. Give the value to compare against"); + return Condition.faulted(); + } + + ValueType type; + if (operator.takesList()) { + if (operand.getKind() != ScriptNode.Kind.LIST) { + fault(path + "." + operator, raw, operator + " expects a " + + "list of values, found " + typeOf(operand)); + return Condition.faulted(); + } + if (operand.getItems().isEmpty()) { + fault(path + "." + operator, raw, + operator + " expects a non empty list"); + return Condition.faulted(); + } + // A member with no type, such as a mapping, joins as an empty + // string, as JavaScript joins a null. + List memberTypes = new ArrayList<>(); + boolean numeric = true; + boolean allSame = true; + for (ScriptNode member : operand.getItems()) { + if (member.getKind() == ScriptNode.Kind.NULL) { + fault(path + "." + operator, raw, + "a null literal is not allowed in a list"); + return Condition.faulted(); + } + ValueType memberType = inferType(member); + String text = memberType == null ? "" : memberType.toString(); + memberTypes.add(text); + if (memberType != ValueType.INT && + memberType != ValueType.DOUBLE) { + numeric = false; + } + if (text.equals(memberTypes.get(0)) == false) { + allSame = false; + } + } + if (numeric == false && allSame == false) { + fault(path + "." + operator, raw, "every member of a list " + + "must be of the same type, found " + + String.join(", ", memberTypes)); + return Condition.faulted(); + } + if (numeric) { + type = memberTypes.contains(ValueType.DOUBLE.toString()) ? + ValueType.DOUBLE : ValueType.INT; + } else { + type = inferType(operand.getItems().get(0)); + } + } else { + type = inferType(operand); + if (type == null) { + fault(path + "." + operator, raw, "the literal " + + jsonOf(operand) + " has no type format 1 knows"); + return Condition.faulted(); + } + } + + if (type == null || operator.getTypes().contains(type) == false) { + List allowed = new ArrayList<>(); + for (ValueType t : operator.getTypes()) { + allowed.add(t.toString()); + } + fault(path, raw, "operator '" + operator + "' is not allowed on " + + "type " + (type == null ? "null" : type.toString()) + + ". It is allowed on " + String.join(", ", allowed)); + return Condition.faulted(); + } + + int slot = useProperty(property.getText(), type, path, raw); + return Condition.comparison( + slot, property.getText(), operator, operand, type); + } + + private Condition readCheckReference( + ScriptNode raw, + String path, + List checkNames) { + if (raw.getKeys().size() != 1) { + fault(path, raw, "Check must be the only key of its condition, " + + "found " + String.join(", ", raw.getKeys())); + } + ScriptNode name = raw.get("Check"); + if (name.getKind() != ScriptNode.Kind.STRING) { + fault(path + ".Check", raw, + "Check expected the name of a check, found " + typeOf(name)); + return Condition.faulted(); + } + int index = checkNames.indexOf(name.getText()); + if (index < 0) { + fault(path + ".Check", raw, "check '" + name.getText() + + "' is not defined. The checks are " + listOrNone(checkNames)); + return Condition.faulted(); + } + return Condition.checkReference(name.getText(), index); + } + + private Condition readAggregate( + ScriptNode raw, + String path, + List checkNames) { + List aggregateKeys = new ArrayList<>(); + for (String key : raw.getKeys()) { + if (canonicalName(AGGREGATE_NAMES, key) != null) { + aggregateKeys.add(key); + } + } + if (aggregateKeys.size() > 1) { + fault(path, raw, "an aggregate condition takes one of " + + String.join(", ", AGGREGATE_NAMES) + ", found " + + String.join(", ", aggregateKeys)); + return Condition.faulted(); + } + String aggregateKey = aggregateKeys.get(0); + String aggregate = canonicalName(AGGREGATE_NAMES, aggregateKey); + List groupNames = new ArrayList<>(); + int[] group = readGroup(raw.valueOf(aggregateKey), + path + "." + aggregate, raw, checkNames, groupNames); + + List operatorKeys = new ArrayList<>(); + for (String key : raw.getKeys()) { + if (key.equals(aggregateKey) == false) { + operatorKeys.add(key); + } + } + if (operatorKeys.isEmpty()) { + fault(path, raw, "an aggregate condition has no operator. " + + "Expected exactly one of " + + String.join(", ", COUNT_OPERATORS)); + return Condition.faulted(); + } + if (operatorKeys.size() > 1) { + fault(path, raw, "a condition takes exactly one operator, found " + + String.join(", ", operatorKeys)); + return Condition.faulted(); + } + Operator operator = Operator.fromText(operatorKeys.get(0)); + if (operator == null) { + fault(path, raw, "unknown operator '" + operatorKeys.get(0) + + "', expected one of " + String.join(", ", COUNT_OPERATORS)); + return Condition.faulted(); + } + // In and NotIn are refused as well, because a count is compared + // with one whole number, never with a list. + if (COUNT_OPERATORS.contains(operator.toString()) == false) { + fault(path, raw, "operator '" + operator + "' is not allowed " + + "on a count, which is an int"); + return Condition.faulted(); + } + // A count is compared with a whole number and with nothing else. + ScriptNode operand = raw.valueOf(operatorKeys.get(0)); + if (operand.getKind() != ScriptNode.Kind.NUMBER || + isWhole(operand.getNumber()) == false) { + fault(path + "." + operator, raw, "an aggregate is compared " + + "with a whole number, found " + typeOf(operand)); + return Condition.faulted(); + } + return Condition.aggregate( + aggregate.equals("Passed"), + group, + group == null ? null : groupNames, + operator, + operand); + } + + /** + * Read the word Checks, meaning every check, which gives a null group, + * or a list of check names, which gives their indexes. + */ + private int[] readGroup( + ScriptNode raw, + String path, + ScriptNode at, + List checkNames, + List groupNames) { + if (raw.getKind() == ScriptNode.Kind.STRING) { + if (raw.getText().equalsIgnoreCase("Checks")) { + return null; + } + fault(path, at, "a group is the word Checks, meaning every " + + "check, or a list of check names. Found '" + raw.getText() + + "'"); + return new int[0]; + } + if (raw.getKind() == ScriptNode.Kind.LIST) { + List indexes = new ArrayList<>(); + for (ScriptNode name : raw.getItems()) { + if (name.getKind() != ScriptNode.Kind.STRING) { + fault(path, at, "a group lists check names, found " + + typeOf(name)); + continue; + } + int index = checkNames.indexOf(name.getText()); + if (index < 0) { + fault(path, at, "check '" + name.getText() + "' is not " + + "defined. The checks are " + listOrNone(checkNames)); + continue; + } + indexes.add(index); + groupNames.add(name.getText()); + } + int[] group = new int[indexes.size()]; + for (int i = 0; i < group.length; i++) { + group[i] = indexes.get(i); + } + return group; + } + fault(path, at, "a group is the word Checks or a list of check " + + "names, found " + typeOf(raw)); + return new int[0]; + } + + /** + * Record that a condition names a source property with the type its + * literal infers, and give the slot the property is read from. + */ + private int useProperty( + String name, + ValueType type, + String path, + ScriptNode at) { + String key = name.toLowerCase(Locale.ROOT); + PropertyUse use = uses.get(key); + if (use == null) { + use = new PropertyUse(properties.size(), type, path); + uses.put(key, use); + properties.add(new SourceProperty(name, type)); + return use.slot; + } + if (use.type != type) { + fault(path, at, "'" + name + "' is inferred as " + type + + " here but was already inferred as " + use.type + " at " + + use.typePath + ". Every use of a property must infer the " + + "same type"); + } + return use.slot; + } + + /** + * The type a literal infers. A number infers its type from its value + * rather than the way it is written, so 8, 8.0 and 8e0 all infer int, + * and a whole number outside the range of int infers double. + * @return the type, or null for a literal with no type + */ + static ValueType inferType(ScriptNode node) { + switch (node.getKind()) { + case BOOL: + return ValueType.BOOL; + case NUMBER: + if (isWhole(node.getNumber()) == false) { + return ValueType.DOUBLE; + } + return node.getNumber() >= ValueType.INT_MIN && + node.getNumber() <= ValueType.INT_MAX ? + ValueType.INT : ValueType.DOUBLE; + case STRING: + return ValueType.STRING; + default: + return null; + } + } + + /** + * Whether a literal of one type may stand for a value of another. A + * whole number reads as a double. + */ + private static boolean literalMatches( + ValueType literalType, + ValueType outputType) { + return literalType == outputType || + (literalType == ValueType.INT && outputType == ValueType.DOUBLE); + } + + static boolean isWhole(double value) { + return Double.isInfinite(value) == false && + Double.isNaN(value) == false && + value == Math.floor(value); + } + + /** + * Describe a value in a message. + */ + static String typeOf(ScriptNode node) { + if (node == null) { + return "nothing"; + } + switch (node.getKind()) { + case NULL: + return "a null literal"; + case LIST: + return "a list"; + case BOOL: + return "a boolean"; + case NUMBER: + return isWhole(node.getNumber()) ? "an integer" : "a number"; + case STRING: + return "a string (" + jsonOf(node) + ")"; + default: + return "a mapping"; + } + } + + /** + * A value as JavaScript's template strings print it. + */ + static String displayOf(ScriptNode node) { + switch (node.getKind()) { + case NULL: + return "null"; + case BOOL: + case NUMBER: + return Literal.of(node).toString(); + case STRING: + return node.getText(); + case LIST: + List parts = new ArrayList<>(); + for (ScriptNode item : node.getItems()) { + parts.add(displayOf(item)); + } + return String.join(",", parts); + default: + return "[object Object]"; + } + } + + /** + * A value as compact JSON, as JSON.stringify gives it. + */ + static String jsonOf(ScriptNode node) { + StringBuilder text = new StringBuilder(); + appendJson(text, node); + return text.toString(); + } + + private static void appendJson(StringBuilder text, ScriptNode node) { + switch (node.getKind()) { + case NULL: + text.append("null"); + break; + case BOOL: + text.append(node.getBool() ? "true" : "false"); + break; + case NUMBER: + text.append(Double.isInfinite(node.getNumber()) || + Double.isNaN(node.getNumber()) ? + "null" : JsText.number(node.getNumber())); + break; + case STRING: + text.append(JsText.quote(node.getText())); + break; + case LIST: + text.append('['); + for (int i = 0; i < node.getItems().size(); i++) { + if (i > 0) { + text.append(','); + } + appendJson(text, node.getItems().get(i)); + } + text.append(']'); + break; + default: + text.append('{'); + for (int i = 0; i < node.getKeys().size(); i++) { + if (i > 0) { + text.append(','); + } + text.append(JsText.quote(node.getKeys().get(i))) + .append(':'); + appendJson(text, node.getValues().get(i)); + } + text.append('}'); + break; + } + } + + private static List unknownKeys( + ScriptNode node, + List allowed) { + List unknown = new ArrayList<>(); + for (String key : node.getKeys()) { + if (canonicalName(allowed, key) == null) { + unknown.add(key); + } + } + return unknown; + } + + /** + * The name in names that matches the key without regard to case. + * @return the name, or null where none matches + */ + private static String canonicalName(List names, String key) { + for (String name : names) { + if (name.equalsIgnoreCase(key)) { + return name; + } + } + return null; + } + + private static String listOrNone(List names) { + return names.isEmpty() ? "(none)" : String.join(", ", names); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedValueMetaData.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedValueMetaData.java new file mode 100644 index 000000000..509daa525 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedValueMetaData.java @@ -0,0 +1,52 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +/** + * One entry of a script's Output.Values, being a value the derived property + * can have. + */ +public final class DerivedValueMetaData { + + private final String name; + private final String description; + + DerivedValueMetaData(String name, String description) { + this.name = name; + this.description = description; + } + + /** + * @return the string form of the value, as a rule's value is compared + */ + public String getName() { + return name; + } + + /** + * @return the description, or null where the script gives none + */ + public String getDescription() { + return description; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/IDerivedPropertyElement.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/IDerivedPropertyElement.java new file mode 100644 index 000000000..fbbaff234 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/IDerivedPropertyElement.java @@ -0,0 +1,45 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.derivedproperty.data.IDerivedPropertyData; + +import java.util.List; + +/** + * A Flow Element that produces new property values from property values + * other elements in the Pipeline have already produced, following rules + * written in a script, and writes them under the Element Data key + * {@code derived}. + */ +public interface IDerivedPropertyElement + extends FlowElement { + + /** + * @return the metadata of every script the element holds, in the order + * the scripts were added + */ + List getScripts(); +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/JsText.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/JsText.java new file mode 100644 index 000000000..0fa241d8b --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/JsText.java @@ -0,0 +1,185 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.math.BigDecimal; + +/** + * Text written the way the JavaScript reference tools write it, so that a + * message, a number read as a string and the canonical form read the same + * in every language. + */ +final class JsText { + + /** + * From this size up, and below {@link #SMALLEST_PLAIN}, JavaScript + * writes a number with an exponent. + */ + private static final double LARGEST_PLAIN = 1e21; + + private static final double SMALLEST_PLAIN = 1e-6; + + private static final char LAST_CONTROL = 0x1f; + private static final char VERTICAL_TAB = 0x0b; + private static final char LINE_SEPARATOR = 0x2028; + private static final char PARAGRAPH_SEPARATOR = 0x2029; + private static final char BYTE_ORDER_MARK = 0xfeff; + private static final String HEX_DIGITS = "0123456789abcdef"; + private static final int NIBBLE_BITS = 4; + private static final int NIBBLE_MASK = 0xf; + private static final int UNICODE_ESCAPE_DIGITS = 4; + + private JsText() { + } + + /** + * Write a number as JavaScript's String does. + * @param value the number + * @return the text + */ + static String number(double value) { + if (Double.isNaN(value)) { + return "NaN"; + } + if (Double.isInfinite(value)) { + return value > 0 ? "Infinity" : "-Infinity"; + } + if (value == 0) { + return "0"; + } + BigDecimal decimal = + new BigDecimal(Double.toString(value)).stripTrailingZeros(); + double size = Math.abs(value); + if (size >= SMALLEST_PLAIN && size < LARGEST_PLAIN) { + return decimal.toPlainString(); + } + String digits = decimal.unscaledValue().abs().toString(); + int exponent = digits.length() - 1 - decimal.scale(); + StringBuilder text = new StringBuilder(); + if (value < 0) { + text.append('-'); + } + text.append(digits.charAt(0)); + if (digits.length() > 1) { + text.append('.').append(digits, 1, digits.length()); + } + text.append('e').append(exponent < 0 ? '-' : '+') + .append(Math.abs(exponent)); + return text.toString(); + } + + /** + * Quote a string as JSON.stringify does. + * @param value the string + * @return the quoted string + */ + static String quote(String value) { + StringBuilder text = new StringBuilder(value.length() + 2); + text.append('"'); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + switch (c) { + case '"': + text.append("\\\""); + break; + case '\\': + text.append("\\\\"); + break; + case '\b': + text.append("\\b"); + break; + case '\f': + text.append("\\f"); + break; + case '\n': + text.append("\\n"); + break; + case '\r': + text.append("\\r"); + break; + case '\t': + text.append("\\t"); + break; + default: + if (c <= LAST_CONTROL || isLoneSurrogate(value, i)) { + unicodeEscape(text, c); + } else { + text.append(c); + } + break; + } + } + return text.append('"').toString(); + } + + /** + * Whether a character is white space as JavaScript's trim reads it. + * @param c the character + * @return true for white space or a line terminator + */ + static boolean isWhitespace(char c) { + return c == '\t' || c == '\n' || c == VERTICAL_TAB || c == '\f' || + c == '\r' || c == LINE_SEPARATOR || c == PARAGRAPH_SEPARATOR || + c == BYTE_ORDER_MARK || + Character.getType(c) == Character.SPACE_SEPARATOR; + } + + /** + * Remove leading and trailing white space as JavaScript's trim does. + * @param value the text + * @return the text without the white space around it + */ + static String trim(String value) { + int start = 0; + int end = value.length(); + while (start < end && isWhitespace(value.charAt(start))) { + start++; + } + while (end > start && isWhitespace(value.charAt(end - 1))) { + end--; + } + return value.substring(start, end); + } + + private static boolean isLoneSurrogate(String value, int i) { + char c = value.charAt(i); + if (Character.isHighSurrogate(c)) { + return i + 1 >= value.length() || + Character.isLowSurrogate(value.charAt(i + 1)) == false; + } + if (Character.isLowSurrogate(c)) { + return i == 0 || + Character.isHighSurrogate(value.charAt(i - 1)) == false; + } + return false; + } + + private static void unicodeEscape(StringBuilder text, char c) { + text.append("\\u"); + for (int shift = (UNICODE_ESCAPE_DIGITS - 1) * NIBBLE_BITS; + shift >= 0; + shift -= NIBBLE_BITS) { + text.append(HEX_DIGITS.charAt((c >> shift) & NIBBLE_MASK)); + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Literal.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Literal.java new file mode 100644 index 000000000..3682ccfe4 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Literal.java @@ -0,0 +1,93 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +/** + * A scalar from a script, being a boolean, a number or a string. + */ +final class Literal { + + /** + * What a value the validator refused becomes. The model holding it is + * discarded before anything reads it. + */ + static final Literal NONE = + new Literal(ScriptNode.Kind.NULL, false, 0, ""); + + private final ScriptNode.Kind kind; + private final boolean bool; + private final double number; + private final String text; + + private Literal( + ScriptNode.Kind kind, + boolean bool, + double number, + String text) { + this.kind = kind; + this.bool = bool; + this.number = number; + this.text = text; + } + + static Literal of(ScriptNode node) { + return new Literal( + node.getKind(), + node.getBool(), + node.getNumber(), + node.getText()); + } + + ScriptNode.Kind getKind() { + return kind; + } + + boolean getBool() { + return bool; + } + + double getNumber() { + return number; + } + + String getText() { + return text; + } + + /** + * The string form of the literal, as JavaScript's String gives it, + * which is how a rule's value is compared with Output.Values. + * @return the string form + */ + @Override + public String toString() { + switch (kind) { + case BOOL: + return bool ? "true" : "false"; + case NUMBER: + return JsText.number(number); + default: + return text; + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Operator.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Operator.java new file mode 100644 index 000000000..6deb51f67 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Operator.java @@ -0,0 +1,106 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * The operators of format 1, each with the types it is allowed on. + */ +enum Operator { + EQ("Eq", ValueType.BOOL, ValueType.INT, ValueType.DOUBLE, + ValueType.STRING), + NE("Ne", ValueType.BOOL, ValueType.INT, ValueType.DOUBLE, + ValueType.STRING), + GT("Gt", ValueType.INT, ValueType.DOUBLE), + GE("Ge", ValueType.INT, ValueType.DOUBLE), + LT("Lt", ValueType.INT, ValueType.DOUBLE), + LE("Le", ValueType.INT, ValueType.DOUBLE), + IN("In", ValueType.BOOL, ValueType.INT, ValueType.DOUBLE, + ValueType.STRING), + NOT_IN("NotIn", ValueType.BOOL, ValueType.INT, ValueType.DOUBLE, + ValueType.STRING), + STARTS_WITH("StartsWith", ValueType.STRING), + ENDS_WITH("EndsWith", ValueType.STRING), + CONTAINS("Contains", ValueType.STRING); + + private final String text; + private final List types; + + Operator(String text, ValueType... types) { + this.text = text; + this.types = Collections.unmodifiableList(Arrays.asList(types)); + } + + /** + * @return the types the operator is allowed on + */ + List getTypes() { + return types; + } + + /** + * @return true for In and NotIn, which take a list + */ + boolean takesList() { + return this == IN || this == NOT_IN; + } + + /** + * The operator a key names, matched without regard to case. + * @param key the key in the script + * @return the operator, or null where format 1 has no such operator + */ + static Operator fromText(String key) { + for (Operator operator : values()) { + if (operator.text.equalsIgnoreCase(key)) { + return operator; + } + } + return null; + } + + /** + * @return every operator's name, separated by a comma and a space + */ + static String names() { + StringBuilder names = new StringBuilder(); + for (Operator operator : values()) { + if (names.length() > 0) { + names.append(", "); + } + names.append(operator.text); + } + return names.toString(); + } + + /** + * @return the name format 1 gives the operator, such as Eq + */ + @Override + public String toString() { + return text; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/OutputModel.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/OutputModel.java new file mode 100644 index 000000000..795eb3e88 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/OutputModel.java @@ -0,0 +1,230 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * The Output block of a script, which is the whole definition of the + * derived property. A field the script does not give is null. + */ +final class OutputModel { + + /** + * One entry of Output.Values. + */ + static final class ValueEntry { + private final Literal name; + private final String description; + + ValueEntry(Literal name, String description) { + this.name = name; + this.description = description; + } + + Literal getName() { + return name; + } + + /** + * @return the description, or null where the script gives none + */ + String getDescription() { + return description; + } + } + + /** + * The Output block while the validator reads it. + */ + static final class Draft { + String name = ""; + String description = ""; + ValueType valueType; + String storedValueType; + String defaultValue; + boolean isList; + Boolean isMandatory; + Boolean isObsolete; + String category; + Boolean isPopular; + Boolean exportValues; + String url; + Long displayOrder; + Long propertyId; + ScriptNode vendorIds; + ScriptNode dependencies; + final List values = new ArrayList<>(); + boolean hasValues; + + /** + * Whether the string form of a value is one of the names under + * Output.Values. + */ + boolean listsValue(String value) { + for (ValueEntry entry : values) { + if (entry.getName().toString().equals(value)) { + return true; + } + } + return false; + } + } + + private final String name; + private final String description; + private final ValueType valueType; + private final String storedValueType; + private final String defaultValue; + private final boolean isList; + private final Boolean isMandatory; + private final Boolean isObsolete; + private final String category; + private final Boolean isPopular; + private final Boolean exportValues; + private final String url; + private final Long displayOrder; + private final Long propertyId; + private final ScriptNode vendorIds; + private final ScriptNode dependencies; + private final List computed; + private final List values; + + /** + * @param draft the block as read + * @param computed every source property the script names, used where + * the script gives no Dependencies + */ + OutputModel(Draft draft, List computed) { + this.name = draft.name; + this.description = draft.description; + this.valueType = draft.valueType; + this.storedValueType = draft.storedValueType; + this.defaultValue = draft.defaultValue; + this.isList = draft.isList; + this.isMandatory = draft.isMandatory; + this.isObsolete = draft.isObsolete; + this.category = draft.category; + this.isPopular = draft.isPopular; + this.exportValues = draft.exportValues; + this.url = draft.url; + this.displayOrder = draft.displayOrder; + this.propertyId = draft.propertyId; + this.vendorIds = draft.vendorIds; + this.dependencies = draft.dependencies; + this.computed = Collections.unmodifiableList( + new ArrayList<>(computed)); + this.values = draft.hasValues ? + Collections.unmodifiableList(new ArrayList<>(draft.values)) : + null; + } + + String getName() { + return name; + } + + String getDescription() { + return description; + } + + ValueType getValueType() { + return valueType; + } + + String getStoredValueType() { + return storedValueType; + } + + String getDefaultValue() { + return defaultValue; + } + + boolean isList() { + return isList; + } + + Boolean getIsMandatory() { + return isMandatory; + } + + Boolean getIsObsolete() { + return isObsolete; + } + + String getCategory() { + return category; + } + + Boolean getIsPopular() { + return isPopular; + } + + Boolean getExportValues() { + return exportValues; + } + + String getUrl() { + return url; + } + + Long getDisplayOrder() { + return displayOrder; + } + + Long getPropertyId() { + return propertyId; + } + + /** + * @return the VendorIds list as written, or null + */ + ScriptNode getVendorIds() { + return vendorIds; + } + + /** + * @return the Dependencies list as written, or null where the script + * gives none and {@link #getComputed()} holds them + */ + ScriptNode getDependencies() { + return dependencies; + } + + /** + * @return every source property the checks and the rules name, in + * elementKey.PropertyName form + */ + List getComputed() { + return computed; + } + + /** + * @return the entries of Output.Values, or null where the script gives + * no list + */ + List getValues() { + return values; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ScriptNode.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ScriptNode.java new file mode 100644 index 000000000..800d506e0 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ScriptNode.java @@ -0,0 +1,193 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.Collections; +import java.util.List; + +/** + * One value of a parsed script. A mapping keeps its keys in the order they + * are written, because checks are numbered and faults reported in that + * order. Numbers are held as doubles, because format 1 infers a number's + * type from its value rather than from how it was written. + */ +final class ScriptNode { + + /** + * The kinds of value a node holds. + */ + enum Kind { NULL, BOOL, NUMBER, STRING, LIST, MAP } + + private final Kind kind; + private final boolean bool; + private final double number; + private final String text; + private final List keys; + private final List values; + private final List items; + private final int line; + + private ScriptNode( + Kind kind, + boolean bool, + double number, + String text, + List keys, + List values, + List items, + int line) { + this.kind = kind; + this.bool = bool; + this.number = number; + this.text = text; + this.keys = keys; + this.values = values; + this.items = items; + this.line = line; + } + + static ScriptNode ofNull(int line) { + return new ScriptNode(Kind.NULL, false, 0, "", + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), line); + } + + static ScriptNode ofBool(boolean value, int line) { + return new ScriptNode(Kind.BOOL, value, 0, "", + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), line); + } + + static ScriptNode ofNumber(double value, int line) { + return new ScriptNode(Kind.NUMBER, false, value, "", + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), line); + } + + static ScriptNode ofString(String value, int line) { + return new ScriptNode(Kind.STRING, false, 0, value, + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), line); + } + + static ScriptNode ofList(List items, int line) { + return new ScriptNode(Kind.LIST, false, 0, "", + Collections.emptyList(), + Collections.emptyList(), + Collections.unmodifiableList(items), line); + } + + static ScriptNode ofMap( + List keys, + List values, + int line) { + return new ScriptNode(Kind.MAP, false, 0, "", + Collections.unmodifiableList(keys), + Collections.unmodifiableList(values), + Collections.emptyList(), line); + } + + Kind getKind() { + return kind; + } + + boolean getBool() { + return bool; + } + + double getNumber() { + return number; + } + + /** + * @return the text of a string, or empty for any other kind + */ + String getText() { + return text; + } + + /** + * @return the keys of a mapping, in the order written + */ + List getKeys() { + return keys; + } + + /** + * @return the values of a mapping, in the order of its keys + */ + List getValues() { + return values; + } + + /** + * @return the members of a list + */ + List getItems() { + return items; + } + + /** + * @return the one based line the value starts on, or 0 where unknown + */ + int getLine() { + return line; + } + + boolean isCollection() { + return kind == Kind.MAP || kind == Kind.LIST; + } + + /** + * The value of a mapping's key, matched without regard to case. + * @param name the key + * @return the value, or null where this is not a mapping or has no + * such key + */ + ScriptNode get(String name) { + for (int i = 0; i < keys.size(); i++) { + if (keys.get(i).equalsIgnoreCase(name)) { + return values.get(i); + } + } + return null; + } + + /** + * The value of a mapping's key, matched exactly as written. + * @param key the key + * @return the value, or null where there is no such key + */ + ScriptNode valueOf(String key) { + for (int i = 0; i < keys.size(); i++) { + if (keys.get(i).equals(key)) { + return values.get(i); + } + } + return null; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ScriptParseException.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ScriptParseException.java new file mode 100644 index 000000000..12f245323 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ScriptParseException.java @@ -0,0 +1,45 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +/** + * Text that is not YAML or JSON, or is not a mapping at the top level. + */ +final class ScriptParseException extends Exception { + + private static final long serialVersionUID = 1L; + + private final int line; + + ScriptParseException(String message, int line) { + super(message); + this.line = line; + } + + /** + * @return the one based line, or 0 where the parser gave none + */ + int getLine() { + return line; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Slot.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Slot.java new file mode 100644 index 000000000..ffae41a17 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/Slot.java @@ -0,0 +1,108 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +/** + * One source property as read for one request, converted to the type the + * script infers for it, or the reason it could not be read. + */ +final class Slot { + + private final boolean available; + private final boolean bool; + private final long integer; + private final double real; + private final String text; + private final String reason; + + private Slot( + boolean available, + boolean bool, + long integer, + double real, + String text, + String reason) { + this.available = available; + this.bool = bool; + this.integer = integer; + this.real = real; + this.text = text; + this.reason = reason; + } + + static Slot ofBool(boolean value) { + return new Slot(true, value, 0, 0, null, null); + } + + static Slot ofInt(long value) { + return new Slot(true, false, value, 0, null, null); + } + + static Slot ofDouble(double value) { + return new Slot(true, false, 0, value, null, null); + } + + static Slot ofString(String value) { + return new Slot(true, false, 0, 0, value, null); + } + + static Slot absent(String reason) { + return new Slot(false, false, 0, 0, null, reason); + } + + boolean isAvailable() { + return available; + } + + boolean getBool() { + return bool; + } + + long getInt() { + return integer; + } + + double getDouble() { + return real; + } + + String getString() { + return text; + } + + /** + * @return why the property could not be read, or null where it was + */ + String getReason() { + return reason; + } + + /** + * The value as a number, for a property read as int or double. + * @param type the type the property is read as + * @return the value + */ + double number(ValueType type) { + return type == ValueType.INT ? integer : real; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceProperty.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceProperty.java new file mode 100644 index 000000000..30ff72d30 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceProperty.java @@ -0,0 +1,93 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.Locale; + +/** + * A property another element produces, which a script reads as the type it + * infers from the literals compared with it. + */ +final class SourceProperty { + + private final String key; + private final String name; + private final ValueType type; + private final String elementKey; + private final String writtenKey; + private final String propertyName; + + /** + * @param name the name as the script first writes it, being + * elementKey.PropertyName + * @param type the type inferred for it + */ + SourceProperty(String name, ValueType type) { + int dot = name.indexOf('.'); + this.key = name.toLowerCase(Locale.ROOT); + this.name = name; + this.type = type; + this.writtenKey = name.substring(0, dot); + this.elementKey = writtenKey.toLowerCase(Locale.ROOT); + this.propertyName = name.substring(dot + 1); + } + + /** + * @return the name in lower case, which identifies the property + */ + String getKey() { + return key; + } + + /** + * @return the name as the script first writes it + */ + String getName() { + return name; + } + + ValueType getType() { + return type; + } + + /** + * @return the Element Data key, in lower case + */ + String getElementKey() { + return elementKey; + } + + /** + * @return the Element Data key as the script writes it + */ + String getWrittenKey() { + return writtenKey; + } + + /** + * @return the property half of the name, as written + */ + String getPropertyName() { + return propertyName; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceRead.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceRead.java new file mode 100644 index 000000000..67fcd0786 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceRead.java @@ -0,0 +1,52 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +/** + * One source property, read once per request as one type, however many + * scripts name it. + */ +final class SourceRead { + + private final SourceProperty property; + private final int key; + + /** + * @param property the property and the type it is read as + * @param key the index of the Element Data key among the element's + * keys, or -1 for a derived property, which is read when + * the script that names it runs + */ + SourceRead(SourceProperty property, int key) { + this.property = property; + this.key = key; + } + + SourceProperty getProperty() { + return property; + } + + int getKey() { + return key; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceValues.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceValues.java new file mode 100644 index 000000000..c37a75703 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/SourceValues.java @@ -0,0 +1,383 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.IWeightedValue; +import fiftyone.pipeline.core.data.WktString; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.data.ProvidesValuesAsString; + +import java.math.BigInteger; +import java.util.List; +import java.util.Locale; + +/** + * Reads a source property from the Element Data of the element that + * supplies it, and converts the value to the type the script infers. A + * value arrives as its native type or as a string, and both are accepted, + * but values are never coerced loosely, so N/A, Unknown and the empty + * string never become false or zero. + *

    + * The wording of every reason is that of readSlot in tools/evaluate.mjs in + * the derived-properties repository. See + * format + * 1, absent and invalid source properties. + */ +final class SourceValues { + + /** + * The reason given where the property is not in the Element Data, or + * its value is null. + */ + static final String NOT_PRESENT = "property not present on this request"; + + /** + * Stands for a list whose members are not all weighted values. + */ + private static final Object NOT_WEIGHTED = new Object(); + + /** + * The bits in a long, which a BigInteger must fit inside to be read as + * a whole number. + */ + private static final int LONG_BITS = 63; + + private SourceValues() { + } + + /** + * Read a property from the Element Data of its source element. + * @param data the Element Data, or null where the request carries none + * under the key + * @param property the property + * @return the value, converted, or the reason it could not be read + */ + static Slot read(ElementData data, SourceProperty property) { + if (data == null) { + return notAvailable(property, NOT_PRESENT); + } + Object raw; + try { + raw = readRaw(data, property); + } catch (RuntimeException e) { + return notAvailable(property, NOT_PRESENT); + } + if (raw == null) { + return notAvailable(property, NOT_PRESENT); + } + return readValue(raw, property); + } + + /** + * The value an element holds for a property. Where the script compares + * the property with text and the element can give the string it + * stored, that string is read. A typed accessor has to answer in its + * own type, so a stored Unknown, meaning the 51Degrees JavaScript has + * not run, would otherwise read as False, and a script could never see + * it. + */ + private static Object readRaw(ElementData data, SourceProperty property) { + if (property.getType() == ValueType.STRING && + data instanceof ProvidesValuesAsString) { + return ((ProvidesValuesAsString) data) + .getStoredValueAsString(property.getPropertyName()); + } + return data.get(property.getPropertyName()); + } + + /** + * Convert a value an element holds. A value with no value carries its + * own message, a list of weighted values gives the value with the + * highest weight, and any other list cannot be read where one value is + * needed. + */ + static Slot readValue(Object value, SourceProperty property) { + Object raw = value; + if (raw instanceof AspectPropertyValue) { + AspectPropertyValue wrapped = (AspectPropertyValue) raw; + if (wrapped.hasValue() == false) { + String message = wrapped.getNoValueMessage(); + return notAvailable(property, + message == null || message.isEmpty() ? + NOT_PRESENT : message); + } + raw = wrapped.getValue(); + if (raw == null) { + return notAvailable(property, NOT_PRESENT); + } + } + if (raw instanceof List) { + raw = heaviest((List) raw); + if (raw == NOT_WEIGHTED) { + return Slot.absent( + "held a list where a single value is needed"); + } + if (raw == null) { + return notAvailable(property, NOT_PRESENT); + } + } else if (raw instanceof IWeightedValue) { + raw = ((IWeightedValue) raw).getValue(); + if (raw == null) { + return notAvailable(property, NOT_PRESENT); + } + } + Slot slot = convert(raw, property.getType()); + if (slot != null) { + return slot; + } + return Slot.absent("held '" + display(raw) + + "' which cannot be read as " + property.getType()); + } + + private static Slot notAvailable(SourceProperty property, String detail) { + return Slot.absent("element '" + property.getWrittenKey() + + "' has no value for '" + property.getPropertyName() + "': " + + detail); + } + + /** + * The value with the highest weight from a list of weighted values, the + * first where two weigh the same. + * @return the value, or {@link #NOT_WEIGHTED} where the list is empty or + * any member is not a weighted value + */ + private static Object heaviest(List list) { + if (list.isEmpty()) { + return NOT_WEIGHTED; + } + IWeightedValue best = null; + for (Object member : list) { + if (member instanceof IWeightedValue == false) { + return NOT_WEIGHTED; + } + IWeightedValue weighted = (IWeightedValue) member; + if (best == null || + weighted.getRawWeighting() > best.getRawWeighting()) { + best = weighted; + } + } + return best.getValue(); + } + + /** + * Convert a source value, which arrives as its native type or as a + * string. + * @return the value, or null where it cannot be read as the type + */ + static Slot convert(Object raw, ValueType type) { + if (raw instanceof String) { + return convertString((String) raw, type); + } + if (raw instanceof WktString) { + String text = raw.toString(); + return text == null ? null : convertString(text, type); + } + if (raw instanceof Boolean) { + boolean value = (Boolean) raw; + switch (type) { + case BOOL: + return Slot.ofBool(value); + case STRING: + return Slot.ofString(value ? "True" : "False"); + default: + return null; + } + } + Long whole = wholeNumber(raw); + if (whole != null) { + long value = whole; + switch (type) { + case INT: + return value < ValueType.INT_MIN || + value > ValueType.INT_MAX ? + null : Slot.ofInt(value); + case DOUBLE: + return Slot.ofDouble(value); + case STRING: + return Slot.ofString(Long.toString(value)); + default: + return null; + } + } + if (raw instanceof Double || raw instanceof Float) { + double value = ((Number) raw).doubleValue(); + switch (type) { + case INT: + return DerivedScriptValidator.isWhole(value) == false || + value < ValueType.INT_MIN || + value > ValueType.INT_MAX ? + null : Slot.ofInt((long) value); + case DOUBLE: + return Double.isInfinite(value) || Double.isNaN(value) ? + null : Slot.ofDouble(value); + case STRING: + return Slot.ofString(JsText.number(value)); + default: + return null; + } + } + return null; + } + + /** + * A value of one of Java's whole number types, or null for any other. + */ + private static Long wholeNumber(Object raw) { + if (raw instanceof Integer || raw instanceof Long || + raw instanceof Short || raw instanceof Byte) { + return ((Number) raw).longValue(); + } + if (raw instanceof BigInteger && + ((BigInteger) raw).bitLength() <= LONG_BITS) { + return ((BigInteger) raw).longValue(); + } + return null; + } + + /** + * Convert the string form of a value. + * @return the value, or null where it cannot be read as the type + */ + static Slot convertString(String raw, ValueType type) { + switch (type) { + case BOOL: { + String trimmed = JsText.trim(raw).toLowerCase(Locale.ROOT); + if (trimmed.equals("true")) { + return Slot.ofBool(true); + } + if (trimmed.equals("false")) { + return Slot.ofBool(false); + } + return null; + } + case INT: { + String trimmed = JsText.trim(raw); + if (isIntegerText(trimmed) == false) { + return null; + } + try { + long value = Long.parseLong(trimmed); + return value < ValueType.INT_MIN || + value > ValueType.INT_MAX ? + null : Slot.ofInt(value); + } catch (NumberFormatException e) { + return null; + } + } + case DOUBLE: { + String trimmed = JsText.trim(raw); + if (isDecimalText(trimmed) == false) { + return null; + } + double value = Double.parseDouble(trimmed); + return Double.isInfinite(value) || Double.isNaN(value) ? + null : Slot.ofDouble(value); + } + case STRING: + return Slot.ofString(raw); + default: + return null; + } + } + + /** + * Whether text is an optional sign and then ASCII digits. + */ + private static boolean isIntegerText(String text) { + int start = startAfterSign(text); + return start < text.length() && + digits(text, start) == text.length() - start; + } + + /** + * Whether text is an optional sign, digits with a full stop as the + * decimal separator, and an optional exponent. + */ + private static boolean isDecimalText(String text) { + int position = startAfterSign(text); + int whole = digits(text, position); + position += whole; + int fraction = 0; + if (position < text.length() && text.charAt(position) == '.') { + position++; + fraction = digits(text, position); + position += fraction; + } + if (whole == 0 && fraction == 0) { + return false; + } + if (position < text.length() && + (text.charAt(position) == 'e' || text.charAt(position) == 'E')) { + position++; + if (position < text.length() && + (text.charAt(position) == '+' || + text.charAt(position) == '-')) { + position++; + } + int exponent = digits(text, position); + if (exponent == 0) { + return false; + } + position += exponent; + } + return position == text.length(); + } + + private static int startAfterSign(String text) { + return text.length() > 0 && + (text.charAt(0) == '+' || text.charAt(0) == '-') ? 1 : 0; + } + + /** + * The number of ASCII digits in text from start. + */ + private static int digits(String text, int start) { + int position = start; + while (position < text.length() && + text.charAt(position) >= '0' && text.charAt(position) <= '9') { + position++; + } + return position - start; + } + + /** + * A source value as a message prints it. + */ + private static String display(Object raw) { + if (raw instanceof String) { + return (String) raw; + } + if (raw instanceof Boolean) { + return (Boolean) raw ? "True" : "False"; + } + Long whole = wholeNumber(raw); + if (whole != null) { + return whole.toString(); + } + if (raw instanceof Double || raw instanceof Float) { + return JsText.number(((Number) raw).doubleValue()); + } + return String.valueOf(raw); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ValueType.java b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ValueType.java new file mode 100644 index 000000000..6de338937 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/main/java/fiftyone/pipeline/derivedproperty/flowelements/ValueType.java @@ -0,0 +1,85 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import java.util.Locale; + +/** + * The four value types format 1 has, each with the Java type a value of it + * is held as. + */ +enum ValueType { + STRING("string", String.class), + BOOL("bool", Boolean.class), + INT("int", Integer.class), + DOUBLE("double", Double.class); + + /** + * The smallest value of int, which format 1 fixes at a signed 32 bit + * whole number so that one script gives one answer in every language. + */ + static final long INT_MIN = Integer.MIN_VALUE; + + /** + * The largest value of int. + */ + static final long INT_MAX = Integer.MAX_VALUE; + + private final String text; + private final Class valueClass; + + ValueType(String text, Class valueClass) { + this.text = text; + this.valueClass = valueClass; + } + + /** + * @return the Java type a value of this type is held as + */ + Class getValueClass() { + return valueClass; + } + + /** + * The value type a script names, matched without regard to case. + * @param text the name in the script + * @return the type, or null where format 1 has no such type + */ + static ValueType fromText(String text) { + String lower = text.toLowerCase(Locale.ROOT); + for (ValueType type : values()) { + if (type.text.equals(lower)) { + return type; + } + } + return null; + } + + /** + * @return the name format 1 gives the type, such as int + */ + @Override + public String toString() { + return text; + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/BuiltInScriptTests.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/BuiltInScriptTests.java new file mode 100644 index 000000000..f8bf3014a --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/BuiltInScriptTests.java @@ -0,0 +1,109 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty; + +import fiftyone.pipeline.derivedproperty.flowelements.DerivedPropertyElement; +import fiftyone.pipeline.derivedproperty.flowelements.DerivedPropertyElementBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; +import java.util.TreeMap; +import java.util.TreeSet; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The scripts this package ships are the scripts folder of the + * derived-properties submodule, and {@link BuiltInScript} names every one + * of them, so a script added to the submodule fails this test until it has + * a constant. + */ +public class BuiltInScriptTests { + + /** + * The scripts in the submodule, by name, with their text. + */ + private static Map submoduleScripts() throws IOException { + Path folder = Paths.get("..", "derived-properties", "scripts"); + assertTrue(Files.isDirectory(folder), "The derived-properties " + + "submodule is missing from " + folder.toAbsolutePath() + + ". Run 'git submodule update --init pipeline/derived-properties'."); + Map scripts = new TreeMap<>(); + try (DirectoryStream stream = Files.newDirectoryStream( + folder, "*.{yaml,yml,json}")) { + for (Path file : stream) { + String name = file.getFileName().toString(); + scripts.put( + name.substring(0, name.lastIndexOf('.')), + new String(Files.readAllBytes(file), + StandardCharsets.UTF_8)); + } + } + return scripts; + } + + @Test + public void enumNamesEveryScriptInTheSubmodule() throws IOException { + TreeSet names = new TreeSet<>(); + for (BuiltInScript script : BuiltInScript.values()) { + names.add(script.name()); + } + assertEquals(submoduleScripts().keySet(), names, + "BuiltInScript and the scripts folder of the derived-properties " + + "submodule differ. Add a constant for each script, named " + + "exactly as the script is."); + } + + @Test + public void packagedScriptsAreTheSubmodulesScripts() throws IOException { + Map scripts = submoduleScripts(); + for (BuiltInScript script : BuiltInScript.values()) { + String text = script.readText(); + assertNotNull(text, script + " is not in the package."); + assertEquals(scripts.get(script.name()), text, + "The packaged " + script + " differs from the submodule's."); + assertSame(script, BuiltInScript.fromName(script.name())); + } + } + + @Test + public void everyScriptBuilds() { + for (BuiltInScript script : BuiltInScript.values()) { + DerivedPropertyElement element = new DerivedPropertyElementBuilder( + LoggerFactory.getILoggerFactory()) + .addScript(script) + .build(); + assertEquals(script.name(), element.getScripts().get(0).getName()); + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementConfigurationTests.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementConfigurationTests.java new file mode 100644 index 000000000..c3a17afda --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementConfigurationTests.java @@ -0,0 +1,161 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.configuration.PipelineOptions; +import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.derivedproperty.data.IDerivedPropertyData; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * A configuration file names the builder as DerivedPropertyElement and + * passes built-in script names as Scripts and file paths as ScriptFiles, + * each a comma separated string. + *

    + * See the + * configuration + * section of the specification. + */ +public class DerivedPropertyElementConfigurationTests { + + @TempDir + Path folder; + + /** + * A file name pattern in the folder, built as text, because Windows + * refuses a Path that holds a wildcard. + */ + private String pattern(String fileNamePattern) { + return folder.toString() + File.separator + fileNamePattern; + } + + /** + * Writes the Band script into the folder and gives a pattern matching + * it. + */ + private String bandFiles() throws Exception { + Files.write(folder.resolve("Band.yaml"), + DerivedPropertyElementTests.BAND_YAML.getBytes( + StandardCharsets.UTF_8)); + return pattern("*.yaml"); + } + + /** + * Options read from an XML configuration file that puts two source + * elements in front of a Derived Property Element with the build + * parameters given. + */ + private PipelineOptions options(String buildParameters) + throws Exception { + String xml = "" + + "StubSourceElement" + + "device" + + "BrowserReleaseAge=1,IsCrawler=False" + + "" + + "StubSourceElement" + + "ip" + + "CountryCode=FR" + + "" + + "DerivedPropertyElement" + + "" + buildParameters + "" + + "" + + ""; + Path file = folder.resolve("pipeline.xml"); + Files.write(file, xml.getBytes(StandardCharsets.UTF_8)); + return PipelineOptionsFactory.getOptionsFromFile(file.toFile()); + } + + @Test + public void scriptsAndScriptFilesFromAConfigurationFile() + throws Exception { + PipelineOptions options = options( + " IsGdpr " + + "" + bandFiles() + ""); + try (Pipeline pipeline = + new PipelineBuilder().buildFromConfiguration(options); + FlowData data = pipeline.createFlowData()) { + data.process(); + IDerivedPropertyData derived = + data.get(IDerivedPropertyData.class); + assertEquals(Boolean.TRUE, + derived.getAs("IsGdpr", Boolean.class).getValue()); + assertEquals("New", + derived.getAs("Band", String.class).getValue()); + assertEquals(2, pipeline.getElement(DerivedPropertyElement.class) + .getScripts().size()); + } + } + + @Test + public void listsAreCommaSeparated() throws Exception { + DerivedPropertyElement element = new DerivedPropertyElementBuilder( + LoggerFactory.getILoggerFactory()) + .setScripts("HumanConfidence, IsGdpr,") + .setScriptFiles(" ," + bandFiles()) + .build(); + assertEquals(3, element.getScripts().size()); + } + + @Test + public void anUnknownBuildParameterIsRefused() throws Exception { + PipelineOptions options = options(""); + assertThrows(PipelineConfigurationException.class, + () -> new PipelineBuilder().buildFromConfiguration(options)); + } + + @Test + public void aPatternMatchingNoFileIsRefused() throws Exception { + PipelineOptions options = options("" + + pattern("*.json") + ""); + PipelineConfigurationException e = assertThrows( + PipelineConfigurationException.class, + () -> new PipelineBuilder().buildFromConfiguration(options)); + assertTrue(e.getMessage().contains("no file matches"), + e.getMessage()); + } + + @Test + public void anUnknownBuiltInScriptIsRefused() throws Exception { + PipelineOptions options = options("NoSuchScript"); + PipelineConfigurationException e = assertThrows( + PipelineConfigurationException.class, + () -> new PipelineBuilder().buildFromConfiguration(options)); + assertTrue(e.getMessage().contains( + "'NoSuchScript' is not a built-in script"), e.getMessage()); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementTests.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementTests.java new file mode 100644 index 000000000..3b1d0d63f --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedPropertyElementTests.java @@ -0,0 +1,602 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.derivedproperty.BuiltInScript; +import fiftyone.pipeline.derivedproperty.DerivedScriptValidationException; +import fiftyone.pipeline.derivedproperty.data.IDerivedPropertyData; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.exceptions.NoValueException; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; +import org.slf4j.event.Level; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The Derived Property Element against the specification and format 1, + * using scripts written here so the tests do not depend on the content of + * the shared scripts. + */ +public class DerivedPropertyElementTests { + + static final String BAND_YAML = "Format: 1\n" + + "Name: Band\n" + + "Version: 1.0.0\n" + + "Output:\n" + + " Name: Band\n" + + " Description: A band.\n" + + " ValueType: string\n" + + " IsList: false\n" + + " Values:\n" + + " - { Name: Old }\n" + + " - { Name: New, Description: Recent. }\n" + + "Checks:\n" + + " Recent: { Property: device.BrowserReleaseAge, Lt: 2.0 }\n" + + "Rules:\n" + + " - When: { All: [ { Check: Recent }, " + + "{ Property: device.IsCrawler, Eq: false } ] }\n" + + " Then: New\n" + + " - Else: Old\n"; + + static final String BAND_JSON = "{\"Format\": 1, \"Name\": \"Band\", " + + "\"Version\": \"1.0.0\",\n" + + " \"Output\": {\"Name\": \"Band\", \"Description\": \"A band.\", " + + "\"ValueType\": \"string\", \"IsList\": false,\n" + + " \"Values\": [{\"Name\": \"Old\"}, {\"Name\": \"New\", " + + "\"Description\": \"Recent.\"}]},\n" + + " \"Checks\": {\"Recent\": {\"Property\": " + + "\"device.BrowserReleaseAge\", \"Lt\": 2}},\n" + + " \"Rules\": [{\"When\": {\"All\": [{\"Check\": \"Recent\"}, " + + "{\"Property\": \"device.IsCrawler\", \"Eq\": false}]},\n" + + " \"Then\": \"New\"}, {\"Else\": \"Old\"}]}"; + + /** + * A script reading one property under device and one under ip. + */ + static final String LIKELY_YAML = "Format: 1\n" + + "Name: Likely\n" + + "Version: 1.0.0\n" + + "Output:\n" + + " Name: Likely\n" + + " Description: Whether a person is likely to be viewing the page.\n" + + " ValueType: bool\n" + + " IsList: false\n" + + " Category: Human Detection\n" + + "Checks:\n" + + " NotCrawler: { Property: device.IsCrawler, Eq: false }\n" + + " HumanAddress: { Property: ip.HumanProbability, Ge: 7 }\n" + + "Rules:\n" + + " - When: { Failed: Checks, Eq: 0 }\n" + + " Then: true\n" + + " - Else: false\n"; + + private static final String MOBILE_YAML = "Format: 1\n" + + "Name: Mobile\n" + + "Version: 1.0.0\n" + + "Output: { Name: Mobile, Description: Whether the device is " + + "mobile., ValueType: bool, IsList: false }\n" + + "Rules:\n" + + " - When: { Property: device.IsMobile, Eq: true }\n" + + " Then: true\n" + + " - Else: false\n"; + + private static final String KIND_YAML = "Format: 1\n" + + "Name: Kind\n" + + "Version: 1.0.0\n" + + "Output: { Name: Kind, Description: The kind of device., " + + "ValueType: string, IsList: false }\n" + + "Rules:\n" + + " - When: { Property: derived.Mobile, Eq: true }\n" + + " Then: Phone\n" + + " - Else: Other\n"; + + /** + * A request the Likely script reads as true. + */ + static Map> likelyValues() { + Map> values = new LinkedHashMap<>(); + values.put("device", map("IsCrawler", "False")); + values.put("ip", map("HumanProbability", 9)); + return values; + } + + static Map map(Object... pairs) { + Map map = new LinkedHashMap<>(); + for (int i = 0; i < pairs.length; i += 2) { + map.put((String) pairs[i], pairs[i + 1]); + } + return map; + } + + static DerivedScript mustValidate(String text) { + DerivedScriptValidator.Result result = + DerivedScriptValidator.validate(text, "", "code"); + assertTrue(result.getFaults().isEmpty(), + "The script did not validate:\n" + result.getFaults()); + return result.getScript(); + } + + static DerivedPropertyElementBuilder builder() { + return new DerivedPropertyElementBuilder( + LoggerFactory.getILoggerFactory()); + } + + static DerivedPropertyElement likely() { + return builder().addScript("Likely", LIKELY_YAML).build(); + } + + /** + * A YAML script and the JSON that mirrors it give the same model, and + * the canonical form adds the inferred types and the computed + * dependencies. + */ + @Test + public void yamlAndJsonGiveTheSameModel() { + String fromYaml = CanonicalJson.print(mustValidate(BAND_YAML)); + String fromJson = CanonicalJson.print(mustValidate(BAND_JSON)); + assertEquals(fromYaml, fromJson); + for (String want : new String[]{ + "\"device.BrowserReleaseAge\": {\n \"Type\": \"int\"\n }", + "\"Dependencies\": [\n \"device.BrowserReleaseAge\",\n" + + " \"device.IsCrawler\"\n ]", + "\"Lt\": 2\n"}) { + assertTrue(fromYaml.contains(want), + "The canonical form lacks " + want + ":\n" + fromYaml); + } + } + + /** + * 2 and 2.0 are both whole numbers, so they infer the same type and do + * not conflict. + */ + @Test + public void wholeNumbersInferInt() { + DerivedScript model = mustValidate(BAND_YAML.replace("Checks:\n", + "Checks:\n Young: { Property: device.BrowserReleaseAge, " + + "Lt: 2 }\n")); + assertSame(ValueType.INT, model.getProperties().get(0).getType()); + } + + /** + * Every fault in every script is reported at once, one line each. + */ + @Test + public void buildReportsEveryFault() { + DerivedScriptValidationException e = assertThrows( + DerivedScriptValidationException.class, + () -> builder() + .addScript("First", BAND_YAML.replace( + "Name: Band\nVersion", "Name: First\nVersion: x\nV")) + .addScript("Second", BAND_YAML.replace( + "Format: 1", "Format: 2")) + .setScripts("NoSuchScript") + .build()); + String message = e.getMessage(); + for (String want : new String[]{"First (code)", "Second (code)", + "Format must be 1, found 2", + "'NoSuchScript' is not a built-in script"}) { + assertTrue(message.contains(want), + "The faults lack " + want + ":\n" + message); + } + assertEquals(e.getFaults().size(), message.split("\n").length); + } + + @Test + public void buildNeedsAScript() { + assertThrows(IllegalStateException.class, () -> builder().build()); + } + + @Test + public void twoScriptsWithOneOutputAreRefused() { + DerivedScriptValidationException e = assertThrows( + DerivedScriptValidationException.class, + () -> builder() + .addScript("Band", BAND_YAML) + .addScript("Band", BAND_JSON) + .build()); + assertTrue(e.getMessage().contains("is added more than once"), + e.getMessage()); + assertTrue(e.getMessage().contains( + "is also produced by the script 'Band'"), e.getMessage()); + } + + private static void assertPipelineRefused( + String want, + FlowElement... elements) { + TestPipelineBuilder builder = new TestPipelineBuilder(); + for (FlowElement element : elements) { + builder.addFlowElement(element); + } + PipelineConfigurationException e = assertThrows( + PipelineConfigurationException.class, builder::build); + assertTrue(e.getMessage().contains(want), + "The build gave '" + e.getMessage() + "', not '" + want + "'"); + } + + /** + * The Pipeline check names a key no element has, a source property no + * element lists, and a derived property two elements produce. + */ + @Test + public void pipelineCheckNamesWhatIsMissing() { + List device = Collections.singletonList("IsCrawler"); + assertPipelineRefused( + "no element in the pipeline has the element data key 'ip'", + new StubSourceElement("device", device, null), + likely()); + assertPipelineRefused( + "no element under 'ip' lists the property 'HumanProbability'", + new StubSourceElement("device", device, null), + new StubSourceElement("ip", + Collections.singletonList("CountryCode"), null), + likely()); + assertPipelineRefused( + "two Derived Property Elements in the pipeline produce the " + + "derived property 'Likely'", + new StubSourceElement("device", device, null), + new StubSourceElement("ip", + Collections.singletonList("HumanProbability"), null), + likely(), + likely()); + } + + /** + * Where an element sits is not judged, so an element added before the + * elements it reads builds, and gives no value naming each property. + */ + @Test + public void positionIsNotJudged() throws Exception { + DerivedPropertyElement element = likely(); + List> elements = new ArrayList<>(); + elements.add(element); + elements.addAll(SharedCasesTests.stubsFor(element, likelyValues())); + TestPipelineBuilder builder = new TestPipelineBuilder(); + for (FlowElement e : elements) { + builder.addFlowElement(e); + } + try (Pipeline pipeline = builder.build(); + FlowData data = pipeline.createFlowData()) { + data.process(); + AspectPropertyValue value = data + .get(IDerivedPropertyData.class) + .getAs("Likely", Boolean.class); + SharedCasesTests.checkMissing(value, + Arrays.asList("device.IsCrawler", "ip.HumanProbability"), + "Likely"); + } + } + + /** + * Sources that run in a parallel group are listed after the elements at + * the top level, and the check still finds them. + */ + @Test + public void pipelineCheckFindsAParallelGroup() throws Exception { + DerivedPropertyElement element = likely(); + List stubs = + SharedCasesTests.stubsFor(element, likelyValues()); + try (Pipeline pipeline = new TestPipelineBuilder() + .addFlowElementsParallel( + stubs.toArray(new FlowElement[0])) + .addFlowElement(element) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + data.process(); + assertEquals(Boolean.TRUE, data.get(IDerivedPropertyData.class) + .getAs("Likely", Boolean.class).getValue()); + } + } + + /** + * Two elements added one after the other both write into the one + * Element Data under derived. They are never put in the same parallel + * group, because both would write that Element Data at once. + */ + @Test + public void twoElementsOneAfterTheOtherShareTheElementData() + throws Exception { + DerivedPropertyElement mobile = + builder().addScript("Mobile", MOBILE_YAML).build(); + DerivedPropertyElement band = + builder().addScript("Band", BAND_YAML).build(); + StubSourceElement device = new StubSourceElement("device", + Arrays.asList("IsMobile", "IsCrawler", "BrowserReleaseAge"), + map("IsMobile", true, "IsCrawler", false, + "BrowserReleaseAge", 1)); + try (Pipeline pipeline = new TestPipelineBuilder() + .addFlowElement(device) + .addFlowElement(mobile) + .addFlowElement(band) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + data.process(); + IDerivedPropertyData derived = + data.get(IDerivedPropertyData.class); + assertEquals(Boolean.TRUE, + derived.getAs("Mobile", Boolean.class).getValue()); + assertEquals("New", + derived.getAs("band", String.class).getValue()); + } + } + + /** + * A request with no Element Data under a key a script reads gives no + * value, and the error is logged once for the script. + */ + @Test + public void noElementDataIsLoggedOnce() throws Exception { + RecordingLoggerFactory logs = new RecordingLoggerFactory(Level.INFO); + DerivedPropertyElement element = + new DerivedPropertyElementBuilder(logs) + .addScript("Likely", LIKELY_YAML) + .build(); + try (Pipeline pipeline = new TestPipelineBuilder() + .addFlowElement(new StubSourceElement("device", + Collections.singletonList("IsCrawler"), + map("IsCrawler", false))) + .addFlowElement(new StubSourceElement("ip", + Collections.singletonList("HumanProbability"), + map("HumanProbability", 9), false)) + .addFlowElement(element) + .build()) { + for (int request = 0; request < 3; request++) { + try (FlowData data = pipeline.createFlowData()) { + data.process(); + AspectPropertyValue value = data + .get(IDerivedPropertyData.class) + .getAs("Likely", Boolean.class); + assertFalse(value.hasValue()); + assertTrue(value.getNoValueMessage().contains( + "'ip.HumanProbability' (element 'ip' has no value " + + "for 'HumanProbability': property not present " + + "on this request)"), + value.getNoValueMessage()); + } + } + } + List errors = logs.messages(Level.ERROR); + assertEquals(1, errors.size(), errors.toString()); + assertTrue(errors.get(0).contains("reads 'ip.HumanProbability'"), + errors.get(0)); + assertTrue(errors.get(0).contains( + "after the element that supplies 'ip'"), errors.get(0)); + assertTrue(errors.get(0).contains("reported once"), errors.get(0)); + } + + /** + * A source property absent for any other reason, such as one the + * JavaScript has not populated yet, is not logged. + */ + @Test + public void anAbsentPropertyIsNotLogged() throws Exception { + RecordingLoggerFactory logs = new RecordingLoggerFactory(Level.INFO); + DerivedPropertyElement element = + new DerivedPropertyElementBuilder(logs) + .addScript("Likely", LIKELY_YAML) + .build(); + Map> values = likelyValues(); + values.get("ip").remove("HumanProbability"); + AspectPropertyValue value = + SharedCasesTests.process(element, values, "Likely"); + assertFalse(value.hasValue()); + assertTrue(logs.messages(Level.ERROR).isEmpty(), + logs.messages(Level.ERROR).toString()); + } + + /** + * A script can read the property another script in the same element + * produces, under the key derived. + */ + @Test + public void derivedFromDerived() throws Exception { + DerivedPropertyElement element = builder() + .addScript("Mobile", MOBILE_YAML) + .addScript("Kind", KIND_YAML) + .build(); + Map> values = new LinkedHashMap<>(); + values.put("device", map("IsMobile", "True")); + AspectPropertyValue kind = + SharedCasesTests.process(element, values, "Kind"); + assertTrue(kind.hasValue(), kind.getNoValueMessage()); + assertEquals("Phone", kind.getValue()); + } + + /** + * The element accepts no evidence, lists one property per script and + * exposes each script's full metadata. + */ + @Test + public void metadata() { + DerivedPropertyElement element = builder() + .addScript("Band", BAND_YAML) + .addScript("Likely", LIKELY_YAML) + .build(); + assertFalse(element.getEvidenceKeyFilter() + .include("header.user-agent")); + assertEquals("derived", element.getElementDataKey()); + List properties = element.getProperties(); + assertEquals(2, properties.size()); + assertEquals("Band", properties.get(0).getName()); + assertEquals("", properties.get(0).getCategory()); + assertSame(String.class, properties.get(0).getType()); + assertTrue(properties.get(0).isAvailable()); + assertSame(element, properties.get(0).getElement()); + assertEquals("Likely", properties.get(1).getName()); + assertEquals("Human Detection", properties.get(1).getCategory()); + assertSame(Boolean.class, properties.get(1).getType()); + + List scripts = element.getScripts(); + assertEquals(2, scripts.size()); + DerivedPropertyMetaData band = scripts.get(0); + assertEquals("Band", band.getName()); + assertEquals("1.0.0", band.getVersion()); + assertEquals(1, band.getFormat()); + assertEquals("code", band.getSource()); + assertFalse(band.isDeprecated()); + DerivedOutputMetaData output = band.getOutput(); + assertEquals("A band.", output.getDescription()); + assertEquals("string", output.getValueType()); + assertSame(String.class, output.getValueClass()); + assertFalse(output.isList()); + assertNull(output.getDefaultValue()); + assertNull(output.getCategory()); + assertEquals( + Arrays.asList("device.BrowserReleaseAge", + "device.IsCrawler"), + output.getDependencies()); + List names = new ArrayList<>(); + List descriptions = new ArrayList<>(); + for (DerivedValueMetaData value : output.getValues()) { + names.add(value.getName()); + descriptions.add(value.getDescription()); + } + assertEquals(Arrays.asList("Old", "New"), names); + assertEquals(Arrays.asList(null, "Recent."), descriptions); + assertNull(scripts.get(1).getOutput().getValues()); + } + + /** + * A built-in script's source is its name. + */ + @Test + public void builtInScriptSourceIsItsName() { + DerivedPropertyElement element = builder() + .addScript(BuiltInScript.IsGdpr) + .build(); + assertEquals("IsGdpr", element.getScripts().get(0).getSource()); + } + + /** + * The build logs one information line per script, the compiled model + * at debug level and a warning carrying a deprecated script's note. + */ + @Test + public void buildLogs() { + RecordingLoggerFactory logs = new RecordingLoggerFactory(Level.DEBUG); + String deprecated = BAND_YAML.replace("Version: 1.0.0\n", + "Version: 1.0.0\nDeprecated: true\n" + + "DeprecationNote: Use Likely.\n"); + new DerivedPropertyElementBuilder(logs) + .addScript("Likely", LIKELY_YAML) + .addScript("Band", deprecated) + .build(); + List info = logs.messages(Level.INFO, "Derived property"); + assertEquals(2, info.size(), info.toString()); + assertEquals("Derived property script 'Likely' version 1.0.0, " + + "format 1, from code, produces 'derived.Likely'", info.get(0)); + List debug = logs.messages(Level.DEBUG, "Derived property"); + assertEquals(2, debug.size(), debug.toString()); + assertTrue(debug.get(0).contains("\"Name\": \"Likely\""), + debug.get(0)); + assertTrue(debug.get(0).contains( + "\"ip.HumanProbability\": {\n \"Type\": \"int\"\n }"), + debug.get(0)); + List warnings = logs.messages(Level.WARN, "Derived property"); + assertEquals(1, warnings.size(), warnings.toString()); + assertEquals("Derived property script 'Band' is deprecated. " + + "Use Likely.", warnings.get(0)); + } + + /** + * The compiled model is printed only where debug is on. + */ + @Test + public void canonicalJsonIsNotWrittenWithoutDebug() { + RecordingLoggerFactory logs = new RecordingLoggerFactory(Level.INFO); + new DerivedPropertyElementBuilder(logs) + .addScript("Likely", LIKELY_YAML) + .build(); + assertTrue(logs.messages(Level.DEBUG).isEmpty()); + assertEquals(1, logs.messages(Level.INFO, "Derived property").size()); + } + + /** + * The Element Data gives a value of the type the script produces, and + * refuses another type or a property no script produces. + */ + @Test + public void elementDataGetAs() throws Exception { + DerivedPropertyElement element = likely(); + try (Pipeline pipeline = buildWithStubs(element, likelyValues()); + FlowData data = pipeline.createFlowData()) { + data.process(); + IDerivedPropertyData derived = data.get(IDerivedPropertyData.class); + assertEquals(Boolean.TRUE, + derived.getAs("likely", Boolean.class).getValue()); + assertThrows(ClassCastException.class, + () -> derived.getAs("Likely", String.class)); + assertThrows(PropertyMissingException.class, + () -> derived.getAs("Unlikely", Boolean.class)); + } + } + + /** + * Reading a value with no value raises the existing no value error, + * carrying the message. + */ + @Test + public void readingNoValueThrowsWithTheMessage() throws Exception { + Map> values = likelyValues(); + values.get("device").put("IsCrawler", "N/A"); + AspectPropertyValue value = + SharedCasesTests.process(likely(), values, "Likely"); + NoValueException e = + assertThrows(NoValueException.class, value::getValue); + assertEquals("Derived property 'Likely' has no value because 1 " + + "source property was not available. 'device.IsCrawler' (held " + + "'N/A' which cannot be read as bool). " + + DerivedPropertyElement.USUAL_CAUSES, e.getMessage()); + } + + static Pipeline buildWithStubs( + DerivedPropertyElement element, + Map> values) throws Exception { + TestPipelineBuilder builder = new TestPipelineBuilder(); + for (StubSourceElement stub : + SharedCasesTests.stubsFor(element, values)) { + builder.addFlowElement(stub); + } + return builder.addFlowElement(element).build(); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptParserTests.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptParserTests.java new file mode 100644 index 000000000..5e73f77da --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/DerivedScriptParserTests.java @@ -0,0 +1,216 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.derivedproperty.DerivedScriptFault; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * How a script's text is read. Plain scalars follow the YAML 1.2 core + * schema, as the reference tools do, quoted scalars are strings, keys are + * written once without regard to case, and JSON is read by the same + * reader. + *

    + * See writing + * YAML that reads the way you meant. + */ +public class DerivedScriptParserTests { + + private static ScriptNode value(String yaml) throws Exception { + return DerivedScriptParser.parse("Value: " + yaml).get("Value"); + } + + /** + * A script comparing device.Flag with the literal given. + */ + private static String flagScript(String literal) { + return "Format: 1\nName: Flag\nVersion: \"1.0.0\"\n" + + "Output: { Name: Flag, Description: A flag., ValueType: bool, " + + "IsList: false }\n" + + "Rules:\n" + + " - When: { Property: device.Flag, Eq: " + literal + " }\n" + + " Then: true\n" + + " - Else: false\n"; + } + + private static ValueType inferred(String literal) { + return DerivedPropertyElementTests.mustValidate(flagScript(literal)) + .getProperties().get(0).getType(); + } + + @Test + public void onlyTrueAndFalseAreBooleans() throws Exception { + for (String text : new String[]{"true", "True", "TRUE"}) { + assertSame(ScriptNode.Kind.BOOL, value(text).getKind(), text); + assertTrue(value(text).getBool(), text); + } + for (String text : new String[]{"false", "False", "FALSE"}) { + assertSame(ScriptNode.Kind.BOOL, value(text).getKind(), text); + } + for (String text : new String[]{"yes", "no", "on", "off", "y", "n", + "NO", "Yes", "tRUE"}) { + assertSame(ScriptNode.Kind.STRING, value(text).getKind(), text); + assertEquals(text, value(text).getText()); + } + } + + @Test + public void yesInfersStringAndTrueInfersBool() { + assertSame(ValueType.STRING, inferred("yes")); + assertSame(ValueType.STRING, inferred("\"true\"")); + assertSame(ValueType.STRING, inferred("'false'")); + assertSame(ValueType.BOOL, inferred("true")); + } + + @Test + public void numbersFollowTheCoreSchema() throws Exception { + assertEquals(8, value("8").getNumber()); + assertEquals(-8, value("-8").getNumber()); + assertEquals(8, value("+8").getNumber()); + assertEquals(8.5, value("8.5").getNumber()); + assertEquals(800, value("8e2").getNumber()); + assertEquals(0.5, value(".5").getNumber()); + assertEquals(16, value("0x10").getNumber()); + assertEquals(15, value("0o17").getNumber()); + assertEquals(Double.POSITIVE_INFINITY, value(".inf").getNumber()); + assertEquals(Double.NEGATIVE_INFINITY, value("-.Inf").getNumber()); + assertTrue(Double.isNaN(value(".NaN").getNumber())); + for (String text : new String[]{"1_000", "0b101", "1:30", "8.5.1", + "0x", "1e", "0o8"}) { + assertSame(ScriptNode.Kind.STRING, value(text).getKind(), text); + } + assertSame(ScriptNode.Kind.STRING, value("\"8\"").getKind()); + } + + @Test + public void wholeNumbersInferIntWhateverTheirForm() { + assertSame(ValueType.INT, inferred("8")); + assertSame(ValueType.INT, inferred("8.0")); + assertSame(ValueType.INT, inferred("8e0")); + assertSame(ValueType.DOUBLE, inferred("8.5")); + // Outside the range of int a whole number infers double. + assertSame(ValueType.DOUBLE, inferred("3000000000")); + } + + @Test + public void nullForms() throws Exception { + for (String text : new String[]{"~", "null", "Null", "NULL", ""}) { + assertSame(ScriptNode.Kind.NULL, value(text).getKind(), text); + } + assertSame(ScriptNode.Kind.STRING, value("'null'").getKind()); + } + + @Test + public void aKeyWrittenTwiceIsRefused() { + ScriptParseException exact = assertThrows(ScriptParseException.class, + () -> DerivedScriptParser.parse("Name: A\nName: B\n")); + assertTrue(exact.getMessage().contains( + "the key 'Name' is written twice in one mapping"), + exact.getMessage()); + ScriptParseException cased = assertThrows(ScriptParseException.class, + () -> DerivedScriptParser.parse( + "Output:\n Name: A\n name: B\n")); + assertTrue(cased.getMessage().contains( + "the keys 'Name' and 'name' differ only in case"), + cased.getMessage()); + assertEquals(2, cased.getLine()); + ScriptParseException json = assertThrows(ScriptParseException.class, + () -> DerivedScriptParser.parse("{\"Name\": \"A\", \"NAME\": 1}")); + assertTrue(json.getMessage().contains("differ only in case"), + json.getMessage()); + } + + @Test + public void textThatIsNotAScript() { + assertEquals("the script is empty", assertThrows( + ScriptParseException.class, + () -> DerivedScriptParser.parse("")).getMessage()); + assertEquals("the script is empty", assertThrows( + ScriptParseException.class, + () -> DerivedScriptParser.parse("# only a comment\n")) + .getMessage()); + assertEquals("the script must be a mapping of keys to values at the " + + "top level", assertThrows(ScriptParseException.class, + () -> DerivedScriptParser.parse("- 1\n- 2\n")).getMessage()); + ScriptParseException yaml = assertThrows(ScriptParseException.class, + () -> DerivedScriptParser.parse("Name: A\n Bad: [\n")); + assertTrue(yaml.getMessage().startsWith("the text is not valid YAML: "), + yaml.getMessage()); + assertTrue(yaml.getLine() > 0); + ScriptParseException json = assertThrows(ScriptParseException.class, + () -> DerivedScriptParser.parse(" {\"Name\": \"A\"")); + assertTrue(json.getMessage().startsWith("the text is not valid JSON: "), + json.getMessage()); + } + + @Test + public void anAliasThatRefersToItselfIsRefused() { + ScriptParseException e = assertThrows(ScriptParseException.class, + () -> DerivedScriptParser.parse("Rules: &a [ *a ]\n")); + assertTrue(e.getMessage().startsWith("the text is not valid YAML"), + e.getMessage()); + } + + @Test + public void aliasesAreFollowed() throws Exception { + ScriptNode document = DerivedScriptParser.parse( + "A: &x { Property: device.X, Eq: 1 }\nB: *x\n"); + assertEquals(1, document.get("B").get("Eq").getNumber()); + } + + @Test + public void faultsCarryTheLineOfTheMapping() { + DerivedScriptValidator.Result result = DerivedScriptValidator.validate( + DerivedPropertyElementTests.BAND_YAML.replace( + "{ Property: device.IsCrawler, Eq: false }", + "{ Property: device.IsCrawler, Equals: false }"), + "Band", "code"); + assertNull(result.getScript()); + assertEquals(1, result.getFaults().size(), result.getFaults() + .toString()); + DerivedScriptFault fault = result.getFaults().get(0); + assertEquals("Rules[0].When.All[1]", fault.getPath()); + assertEquals(15, fault.getLine()); + assertEquals("Band (code) line 15 at Rules[0].When.All[1]: unknown " + + "operator 'Equals', expected one of Eq, Ne, Gt, Ge, Lt, Le, In, " + + "NotIn, StartsWith, EndsWith, Contains", fault.toString()); + } + + @Test + public void inAndNotInAreRefusedOnACount() { + DerivedScriptValidator.Result result = DerivedScriptValidator.validate( + DerivedPropertyElementTests.LIKELY_YAML.replace( + "{ Failed: Checks, Eq: 0 }", "{ Failed: Checks, In: 0 }"), + "Likely", "code"); + assertNull(result.getScript()); + assertTrue(result.getFaults().get(0).getMessage().contains( + "operator 'In' is not allowed on a count"), + result.getFaults().toString()); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/RecordingLoggerFactory.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/RecordingLoggerFactory.java new file mode 100644 index 000000000..1f60e09e9 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/RecordingLoggerFactory.java @@ -0,0 +1,147 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; +import org.slf4j.Marker; +import org.slf4j.event.Level; +import org.slf4j.helpers.LegacyAbstractLogger; +import org.slf4j.helpers.MessageFormatter; + +import java.util.ArrayList; +import java.util.List; + +/** + * Keeps every message logged at or above a level, so a test can read what + * the element logged. + */ +final class RecordingLoggerFactory implements ILoggerFactory { + + private final Level threshold; + private final List levels = new ArrayList<>(); + private final List messages = new ArrayList<>(); + + RecordingLoggerFactory(Level threshold) { + this.threshold = threshold; + } + + @Override + public Logger getLogger(String name) { + return new RecordingLogger(name, this); + } + + /** + * @param level the level + * @return every message logged at exactly the level, in order + */ + synchronized List messages(Level level) { + List found = new ArrayList<>(); + for (int i = 0; i < levels.size(); i++) { + if (levels.get(i) == level) { + found.add(messages.get(i)); + } + } + return found; + } + + /** + * @param level the level + * @param start the text the message starts with + * @return every message logged at the level that starts with the text + */ + List messages(Level level, String start) { + List found = new ArrayList<>(); + for (String message : messages(level)) { + if (message.startsWith(start)) { + found.add(message); + } + } + return found; + } + + private boolean enabled(Level level) { + return level.toInt() >= threshold.toInt(); + } + + private synchronized void record(Level level, String message) { + levels.add(level); + messages.add(message); + } + + /** + * A logger that hands every message to the factory. + */ + private static final class RecordingLogger extends LegacyAbstractLogger { + + private static final long serialVersionUID = 1L; + + private final transient RecordingLoggerFactory factory; + + RecordingLogger(String name, RecordingLoggerFactory factory) { + this.name = name; + this.factory = factory; + } + + @Override + public boolean isTraceEnabled() { + return factory.enabled(Level.TRACE); + } + + @Override + public boolean isDebugEnabled() { + return factory.enabled(Level.DEBUG); + } + + @Override + public boolean isInfoEnabled() { + return factory.enabled(Level.INFO); + } + + @Override + public boolean isWarnEnabled() { + return factory.enabled(Level.WARN); + } + + @Override + public boolean isErrorEnabled() { + return factory.enabled(Level.ERROR); + } + + @Override + protected String getFullyQualifiedCallerName() { + return null; + } + + @Override + protected void handleNormalizedLoggingCall( + Level level, + Marker marker, + String messagePattern, + Object[] arguments, + Throwable throwable) { + factory.record(level, + MessageFormatter.basicArrayFormat(messagePattern, arguments)); + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/SharedCasesTests.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/SharedCasesTests.java new file mode 100644 index 000000000..858202621 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/SharedCasesTests.java @@ -0,0 +1,419 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.IWeightedValue; +import fiftyone.pipeline.core.data.WeightedValue; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.derivedproperty.BuiltInScript; +import fiftyone.pipeline.derivedproperty.DerivedScriptFault; +import fiftyone.pipeline.derivedproperty.data.IDerivedPropertyData; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Runs the case files of the derived-properties repository, which every + * language runs, so the answers and the faults are checked to match case + * for case. The repository is the submodule pipeline/derived-properties. + *

    + * See writing + * and running cases. + */ +public class SharedCasesTests { + + /** + * The largest raw weighting a weighted value carries. + */ + private static final int MAX_RAW_WEIGHTING = 65535; + + /** + * The derived-properties submodule, which a test fails without. + */ + static Path sharedFolder() { + Path folder = Paths.get("..", "derived-properties"); + if (Files.isDirectory(folder.resolve("tests")) == false) { + fail("The derived-properties submodule is missing from " + + folder.toAbsolutePath() + ". Run 'git submodule update " + + "--init pipeline/derived-properties'."); + } + return folder; + } + + /** + * The files in a folder whose names end with the text given, in name + * order. + */ + static List files(Path folder, String glob) throws IOException { + List found = new ArrayList<>(); + try (DirectoryStream stream = + Files.newDirectoryStream(folder, glob)) { + for (Path path : stream) { + found.add(path); + } + } + found.sort((a, b) -> a.getFileName().toString() + .compareTo(b.getFileName().toString())); + return found; + } + + static String read(Path file) throws IOException { + return new String(Files.readAllBytes(file), StandardCharsets.UTF_8); + } + + @TestFactory + public List sharedCases() throws Exception { + List files = files( + sharedFolder().resolve("tests"), "*.cases.yaml"); + assertFalse(files.isEmpty(), + "There are no case files in the derived-properties submodule."); + List tests = new ArrayList<>(); + for (Path file : files) { + ScriptNode cases = DerivedScriptParser.parse(read(file)); + String script = cases.get("Script").getText(); + BuiltInScript builtIn = BuiltInScript.fromName(script); + if (builtIn == null) { + fail(file + " proves the script " + script + ", which has " + + "no BuiltInScript."); + } + DerivedPropertyElement element = + new DerivedPropertyElementBuilder( + LoggerFactory.getILoggerFactory()) + .addScript(builtIn) + .build(); + String output = element.getScripts().get(0).getOutput().getName(); + for (boolean wrap : new boolean[]{false, true}) { + for (ScriptNode c : cases.get("Cases").getItems()) { + String name = script + "/" + c.get("Name").getText() + + "/wrapped=" + wrap; + tests.add(DynamicTest.dynamicTest(name, () -> + runCase(element, output, c, wrap))); + } + } + } + return tests; + } + + private static void runCase( + DerivedPropertyElement element, + String output, + ScriptNode c, + boolean wrap) throws Exception { + Map> values = new LinkedHashMap<>(); + ScriptNode properties = c.get("Properties"); + for (int i = 0; i < properties.getKeys().size(); i++) { + String property = properties.getKeys().get(i); + int dot = property.indexOf('.'); + String key = property.substring(0, dot).toLowerCase(Locale.ROOT); + Map forKey = values.get(key); + if (forKey == null) { + forKey = new LinkedHashMap<>(); + values.put(key, forKey); + } + forKey.put(property.substring(dot + 1), + caseValue(properties.getValues().get(i), wrap)); + } + AspectPropertyValue got = process(element, values, output); + ScriptNode expect = c.get("Expect"); + ScriptNode value = expect.get("Value"); + if (value != null) { + assertTrue(got.hasValue(), "Expected '" + expectedText(value) + + "' but there was no value. " + got.getNoValueMessage()); + assertEquals(expectedText(value), valueText(got.getValue())); + return; + } + List missing = new ArrayList<>(); + for (ScriptNode name : expect.get("Missing").getItems()) { + missing.add(name.getText()); + } + checkMissing(got, missing, output); + } + + /** + * Run one request through stubs holding the values and the element, + * and give what the element wrote for the output property. + */ + static AspectPropertyValue process( + DerivedPropertyElement element, + Map> values, + String output) throws Exception { + TestPipelineBuilder builder = new TestPipelineBuilder(); + for (StubSourceElement stub : stubsFor(element, values)) { + builder.addFlowElement(stub); + } + try (Pipeline pipeline = builder.addFlowElement(element).build(); + FlowData data = pipeline.createFlowData()) { + data.process(); + Object value = data.get(IDerivedPropertyData.class).get(output); + assertTrue(value instanceof AspectPropertyValue, + output + " is " + value + ", not an AspectPropertyValue"); + return (AspectPropertyValue) value; + } + } + + /** + * A stub for each key the element reads, listing every property read + * under it and holding the values given for it. + */ + static List stubsFor( + DerivedPropertyElement element, + Map> values) { + Map> names = new LinkedHashMap<>(); + for (SourceProperty property : element.sourceProperties()) { + List forKey = names.get(property.getElementKey()); + if (forKey == null) { + forKey = new ArrayList<>(); + names.put(property.getElementKey(), forKey); + } + forKey.add(property.getPropertyName()); + } + List stubs = new ArrayList<>(); + for (Map.Entry> key : names.entrySet()) { + stubs.add(new StubSourceElement( + key.getKey(), key.getValue(), values.get(key.getKey()))); + } + return stubs; + } + + /** + * A value from a case file as a source element holds it. The String + * form is the string, NoValue is a value with no value, and a list of + * Value and Weight pairs is a list of weighted values. + */ + private static Object caseValue(ScriptNode node, boolean wrap) { + switch (node.getKind()) { + case NULL: + return null; + case BOOL: + case NUMBER: + case STRING: + Object value = scalar(node); + return wrap ? new AspectPropertyValueDefault<>(value) : value; + case MAP: + ScriptNode noValue = node.get("NoValue"); + if (noValue != null) { + AspectPropertyValueDefault missing = + new AspectPropertyValueDefault<>(); + missing.setNoValueMessage(Literal.of(noValue).toString()); + return missing; + } + ScriptNode text = node.get("String"); + if (text != null) { + if (text.getKind() == ScriptNode.Kind.NULL) { + return null; + } + String string = Literal.of(text).toString(); + return wrap ? + new AspectPropertyValueDefault<>(string) : string; + } + return new LinkedHashMap<>(); + default: + return weighted(node); + } + } + + /** + * A list whose members are not all weighted values cannot be read + * where one value is needed, whatever it holds. A case gives weights + * of any size, and a raw weighting is 16 bits, so each weight is scaled + * against the largest, which keeps the order. + */ + private static List weighted(ScriptNode node) { + List values = new ArrayList<>(); + List weights = new ArrayList<>(); + double largest = 0; + for (ScriptNode item : node.getItems()) { + if (item.getKind() != ScriptNode.Kind.MAP || + item.get("Value") == null) { + List nulls = new ArrayList<>(); + for (int i = 0; i < node.getItems().size(); i++) { + nulls.add(null); + } + return nulls; + } + ScriptNode weight = item.get("Weight"); + double w = weight == null ? 0 : weight.getNumber(); + values.add(caseValue(item.get("Value"), false)); + weights.add(w); + largest = Math.max(largest, w); + } + List list = new ArrayList<>(); + for (int i = 0; i < values.size(); i++) { + int raw = largest > 0 && weights.get(i) > 0 ? + (int) Math.round(weights.get(i) / largest * MAX_RAW_WEIGHTING) : + 0; + IWeightedValue member = + new WeightedValue<>(raw, values.get(i)); + list.add(member); + } + return list; + } + + /** + * A scalar with the type as written, a whole number being an Integer + * where it fits and a Long where it does not. + */ + private static Object scalar(ScriptNode node) { + switch (node.getKind()) { + case BOOL: + return node.getBool(); + case NUMBER: + double number = node.getNumber(); + if (DerivedScriptValidator.isWhole(number) && + number >= Integer.MIN_VALUE && + number <= Integer.MAX_VALUE) { + return (int) number; + } + if (DerivedScriptValidator.isWhole(number) && + Math.abs(number) < Long.MAX_VALUE) { + return (long) number; + } + return number; + default: + return node.getText(); + } + } + + /** + * The text of a value, as the case files compare values. + */ + static String valueText(Object value) { + if (value instanceof Double) { + return JsText.number((Double) value); + } + return String.valueOf(value); + } + + /** + * The text of an expected value in a case file. + */ + private static String expectedText(ScriptNode node) { + return Literal.of(node).toString(); + } + + /** + * A missing value names every property that could not be read, so the + * message gives the count and each name. + */ + static void checkMissing( + AspectPropertyValue got, + List missing, + String output) { + assertFalse(got.hasValue(), () -> "Expected no value naming " + + missing + " but got '" + got.getValue() + "'"); + String message = got.getNoValueMessage(); + String noun = missing.size() == 1 ? + "1 source property was not available" : + missing.size() + " source properties were not available"; + String prefix = "Derived property '" + output + + "' has no value because " + noun + ". "; + assertTrue(message.startsWith(prefix), + "The message '" + message + "' does not start '" + prefix + "'"); + String lower = message.toLowerCase(Locale.ROOT); + for (String name : missing) { + assertTrue( + lower.contains("'" + name.toLowerCase(Locale.ROOT) + "' ("), + "The message '" + message + "' does not name " + name); + } + assertTrue(message.endsWith(DerivedPropertyElement.USUAL_CAUSES), + "The message '" + message + "' does not end with the usual " + + "causes"); + } + + @TestFactory + public List rejectedScripts() throws Exception { + List files = files( + sharedFolder().resolve("tests").resolve("invalid"), "*.yaml"); + assertFalse(files.isEmpty(), "There are no rejection cases in the " + + "derived-properties submodule."); + List tests = new ArrayList<>(); + for (Path file : files) { + String fileName = file.getFileName().toString(); + ScriptNode c = DerivedScriptParser.parse(read(file)); + tests.add(DynamicTest.dynamicTest(fileName, () -> + checkRejected(c, fileName))); + } + return tests; + } + + private static void checkRejected(ScriptNode c, String fileName) { + ScriptNode name = c.get("Name"); + DerivedScriptValidator.Result result = DerivedScriptValidator.validate( + c.get("Script").getText(), + name == null ? "" : name.getText(), + fileName); + assertNull(result.getScript(), + "The script validated, and the case says it must not."); + assertFalse(result.getFaults().isEmpty(), + "The script validated, and the case says it must not."); + StringBuilder all = new StringBuilder(); + for (DerivedScriptFault fault : result.getFaults()) { + all.append('\n').append(fault); + } + ScriptNode expect = c.get("Expect"); + ScriptNode paths = expect.get("Paths"); + if (paths != null) { + for (ScriptNode path : paths.getItems()) { + boolean found = false; + for (DerivedScriptFault fault : result.getFaults()) { + found = found || fault.getPath().equals(path.getText()); + } + assertTrue(found, "Expected a fault at '" + path.getText() + + "'. Faults were:" + all); + } + } + ScriptNode mentions = expect.get("Mentions"); + if (mentions != null) { + for (ScriptNode mention : mentions.getItems()) { + boolean found = false; + for (DerivedScriptFault fault : result.getFaults()) { + found = found || + fault.getMessage().contains(mention.getText()); + } + assertTrue(found, "Expected a fault mentioning '" + + mention.getText() + "'. Faults were:" + all); + } + } + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/SourceValuesTests.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/SourceValuesTests.java new file mode 100644 index 000000000..03aa5edc5 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/SourceValuesTests.java @@ -0,0 +1,293 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.IWeightedValue; +import fiftyone.pipeline.core.data.WeightedValue; +import fiftyone.pipeline.core.data.WktString; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; +import fiftyone.pipeline.engines.data.ProvidesValuesAsString; +import org.junit.jupiter.api.Test; + +import java.math.BigInteger; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * How a source value is converted to the type a script infers, with the + * Java types elements hold. + *

    + * See format + * 1, conversion. + */ +public class SourceValuesTests { + + private static Slot read(Object raw, ValueType type) { + return SourceValues.readValue( + raw, new SourceProperty("device.Value", type)); + } + + private static void assertAbsent(Object raw, ValueType type, String why) { + Slot slot = read(raw, type); + assertFalse(slot.isAvailable(), raw + " was read as " + type); + assertEquals(why, slot.getReason()); + } + + @Test + public void booleans() { + assertTrue(read(true, ValueType.BOOL).getBool()); + assertTrue(read(" TRUE\t", ValueType.BOOL).getBool()); + assertFalse(read("False", ValueType.BOOL).getBool()); + assertEquals("True", read(true, ValueType.STRING).getString()); + assertEquals("False", read(false, ValueType.STRING).getString()); + for (Object raw : new Object[]{"N/A", "Unknown", "", "1", "yes", 1}) { + assertFalse(read(raw, ValueType.BOOL).isAvailable(), + raw + " became a boolean"); + } + assertAbsent("yes", ValueType.BOOL, + "held 'yes' which cannot be read as bool"); + } + + @Test + public void wholeNumbersHaveTheRangeOfInt() { + assertEquals(5, read(5, ValueType.INT).getInt()); + assertEquals(5, read(5L, ValueType.INT).getInt()); + assertEquals(5, read((short) 5, ValueType.INT).getInt()); + assertEquals(5, read((byte) 5, ValueType.INT).getInt()); + assertEquals(5, read(BigInteger.valueOf(5), ValueType.INT).getInt()); + assertEquals(12, read(" +12 ", ValueType.INT).getInt()); + assertEquals(2, read(2.0, ValueType.INT).getInt()); + assertEquals(Integer.MIN_VALUE, + read((long) Integer.MIN_VALUE, ValueType.INT).getInt()); + assertAbsent(3000000000L, ValueType.INT, + "held '3000000000' which cannot be read as int"); + assertAbsent("3000000000", ValueType.INT, + "held '3000000000' which cannot be read as int"); + assertAbsent("99999999999999999999", ValueType.INT, + "held '99999999999999999999' which cannot be read as int"); + assertAbsent(2.5, ValueType.INT, + "held '2.5' which cannot be read as int"); + for (Object raw : new Object[]{"1.0", "Unknown", "", "1e3", "0x10"}) { + assertFalse(read(raw, ValueType.INT).isAvailable(), + raw + " became an int"); + } + } + + @Test + public void doubles() { + assertEquals(1.5, read(1.5f, ValueType.DOUBLE).getDouble()); + assertEquals(3000000000.0, + read(3000000000L, ValueType.DOUBLE).getDouble()); + assertEquals(1000, read("1e3", ValueType.DOUBLE).getDouble()); + assertEquals(0.5, read(".5", ValueType.DOUBLE).getDouble()); + assertEquals(5, read("5.", ValueType.DOUBLE).getDouble()); + assertEquals(-2.5, read(" -2.5E0 ", ValueType.DOUBLE).getDouble()); + for (Object raw : new Object[]{"Unknown", "", "1e400", "NaN", + "Infinity", "0x10", "1,5", Double.NaN, + Double.POSITIVE_INFINITY}) { + assertFalse(read(raw, ValueType.DOUBLE).isAvailable(), + raw + " became a double"); + } + } + + @Test + public void strings() { + assertEquals(" as written ", + read(" as written ", ValueType.STRING).getString()); + assertEquals("8", read(8, ValueType.STRING).getString()); + assertEquals("8", read(8.0, ValueType.STRING).getString()); + assertEquals("8.5", read(8.5, ValueType.STRING).getString()); + assertEquals("1e+21", read(1e21, ValueType.STRING).getString()); + assertEquals("POINT (1 2)", + read(new WktString("POINT (1 2)"), ValueType.STRING).getString()); + } + + @Test + public void aValueWithNoValueCarriesItsMessage() { + AspectPropertyValueDefault noValue = + new AspectPropertyValueDefault<>(); + noValue.setNoValueMessage("The JavaScript has not run yet."); + assertAbsent(noValue, ValueType.BOOL, "element 'device' has no " + + "value for 'Value': The JavaScript has not run yet."); + noValue.setNoValueMessage(""); + assertAbsent(noValue, ValueType.BOOL, "element 'device' has no " + + "value for 'Value': property not present on this request"); + assertTrue(read(new AspectPropertyValueDefault<>("True"), + ValueType.BOOL).getBool()); + } + + @Test + public void aWeightedListGivesTheHeaviestValue() { + List> weighted = + Arrays.>asList( + new WeightedValue<>(100, 2), + new WeightedValue<>(900, 9), + new WeightedValue<>(900, 7)); + assertEquals(9, read(weighted, ValueType.INT).getInt()); + assertEquals(9, read(new AspectPropertyValueDefault<>(weighted), + ValueType.INT).getInt()); + assertEquals(4, read(new WeightedValue<>(1, 4), ValueType.INT) + .getInt()); + assertAbsent(Arrays.asList("a", "b"), ValueType.STRING, + "held a list where a single value is needed"); + assertAbsent(Collections.emptyList(), ValueType.STRING, + "held a list where a single value is needed"); + } + + @Test + public void aMissingPropertyIsNotPresent() { + Slot slot = SourceValues.read(null, + new SourceProperty("Device.Value", ValueType.BOOL)); + assertFalse(slot.isAvailable()); + assertEquals("element 'Device' has no value for 'Value': property " + + "not present on this request", slot.getReason()); + } + + @Test + public void numbersPrintAsJavaScriptPrintsThem() { + assertEquals("0", JsText.number(-0.0)); + assertEquals("1", JsText.number(1.0)); + assertEquals("0.1", JsText.number(0.1)); + assertEquals("123456789012345680000", + JsText.number(1.2345678901234568E20)); + assertEquals("1e+21", JsText.number(1e21)); + assertEquals("1.5e-7", JsText.number(1.5e-7)); + assertEquals("0.000001", JsText.number(1e-6)); + assertEquals("-2.5", JsText.number(-2.5)); + assertEquals("NaN", JsText.number(Double.NaN)); + assertEquals("-Infinity", JsText.number(Double.NEGATIVE_INFINITY)); + } + + @Test + public void stringsQuoteAsJsonStringifyQuotesThem() { + assertEquals("\"a\\\"b\\\\c\\n\\u0001\u00e9\"", + JsText.quote("a\"b\\c\n\u0001\u00e9")); + assertEquals("\"\\ud800\"", JsText.quote("\ud800")); + assertEquals("\"\ud83d\ude00\"", JsText.quote("\ud83d\ude00")); + } + + /** Element Data holding typed values only. */ + private static class Typed implements ElementData { + private final Map typed = new HashMap<>(); + + @Override + public Object get(String key) { + return typed.get(key); + } + + @Override + public void put(String key, Object value) { + typed.put(key, value); + } + + @Override + public void populateFromMap(Map values) { + typed.putAll(values); + } + + @Override + public Map asKeyMap() { + return typed; + } + + @Override + public Pipeline getPipeline() { + return null; + } + + @Override + public void setPipeline(Pipeline pipeline) { + } + } + + /** + * Element Data that can also give the string each value was stored + * as, as the on-premise engines can. + */ + private static final class StoredStrings extends Typed + implements ProvidesValuesAsString { + private final Map stored = new HashMap<>(); + + @Override + public AspectPropertyValue getStoredValueAsString( + String propertyName) { + String value = stored.get(propertyName); + if (value == null) { + AspectPropertyValue none = + new AspectPropertyValueDefault<>(); + none.setNoValueMessage("nothing was stored"); + return none; + } + return new AspectPropertyValueDefault<>(value); + } + } + + /** + * A property the 51Degrees JavaScript has not yet measured is stored + * as Unknown, and its typed value reads False. A script comparing the + * property with text reads the stored string, so it can see Unknown, + * and a script comparing it with a boolean reads the typed value. + */ + @Test + public void storedStringIsReadWhereTheScriptComparesText() { + StoredStrings data = new StoredStrings(); + data.put("IsVisible", new AspectPropertyValueDefault<>(false)); + data.stored.put("IsVisible", "Unknown"); + assertEquals("Unknown", SourceValues.read(data, + new SourceProperty("device.IsVisible", ValueType.STRING)) + .getString()); + assertFalse(SourceValues.read(data, + new SourceProperty("device.IsVisible", ValueType.BOOL)) + .getBool()); + } + + @Test + public void storedStringWithNoValueGivesItsMessage() { + StoredStrings data = new StoredStrings(); + data.put("IsVisible", new AspectPropertyValueDefault<>(true)); + Slot slot = SourceValues.read(data, + new SourceProperty("device.IsVisible", ValueType.STRING)); + assertFalse(slot.isAvailable()); + assertEquals("element 'device' has no value for 'IsVisible': " + + "nothing was stored", slot.getReason()); + } + + @Test + public void typedValueIsReadWhereNoStringIsStored() { + ElementData data = new Typed(); + data.put("IsVisible", new AspectPropertyValueDefault<>(true)); + assertEquals("True", SourceValues.read(data, + new SourceProperty("device.IsVisible", ValueType.STRING)) + .getString()); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/StubSourceElement.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/StubSourceElement.java new file mode 100644 index 000000000..9f3e9bb92 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/StubSourceElement.java @@ -0,0 +1,126 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementDataBase; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.ElementPropertyMetaDataDefault; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.FlowElementBase; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Stands in for an element that supplies source properties, such as device + * detection. It lists every property it is given a name for, as an engine + * lists the properties it can supply, and writes the values it holds under + * its Element Data key. A name with no value is listed but not written. + */ +public final class StubSourceElement + extends FlowElementBase { + + private final String key; + private final List properties; + private final Map values; + private final boolean addData; + + /** + * @param key the Element Data key + * @param names the properties listed + * @param values the values written, by property name + * @param addData false to write no Element Data at all + */ + StubSourceElement( + String key, + Collection names, + Map values, + boolean addData) { + super(LoggerFactory.getLogger(StubSourceElement.class), + (flowData, element) -> new ElementDataBase( + LoggerFactory.getLogger(StubSourceElement.class), + flowData) { + }); + this.key = key; + this.values = values == null ? + Collections.emptyMap() : + new LinkedHashMap<>(values); + List listed = new ArrayList<>(); + for (String name : names) { + listed.add(new ElementPropertyMetaDataDefault( + name, this, "", Object.class, true)); + } + this.properties = Collections.unmodifiableList(listed); + this.addData = addData; + } + + StubSourceElement( + String key, + Collection names, + Map values) { + this(key, names, values, true); + } + + @Override + public String getElementDataKey() { + return key; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist(Collections.emptyList()); + } + + @Override + public List getProperties() { + return properties; + } + + @Override + protected void processInternal(FlowData data) { + if (addData == false) { + return; + } + ElementData elementData = + data.getOrAdd(getTypedDataKey(), getDataFactory()); + for (Map.Entry value : values.entrySet()) { + elementData.put(value.getKey(), value.getValue()); + } + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/StubSourceElementBuilder.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/StubSourceElementBuilder.java new file mode 100644 index 000000000..1acb61d28 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/StubSourceElementBuilder.java @@ -0,0 +1,70 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.annotations.ElementBuilder; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Builds a {@link StubSourceElement}, so that a configuration file can put + * a source element in front of a Derived Property Element. + */ +@ElementBuilder +public class StubSourceElementBuilder { + + private String key = ""; + private final Map values = new LinkedHashMap<>(); + + /** + * @param key the Element Data key the element writes under + * @return this builder + */ + public StubSourceElementBuilder setKey(String key) { + this.key = key; + return this; + } + + /** + * @param values the properties and their string values, written as + * Name=Value and separated by commas + * @return this builder + */ + public StubSourceElementBuilder setValues(String values) { + for (String pair : values.split(",")) { + int equals = pair.indexOf('='); + this.values.put( + pair.substring(0, equals).trim(), + pair.substring(equals + 1).trim()); + } + return this; + } + + /** + * @return a new element + */ + public StubSourceElement build() { + return new StubSourceElement(key, values.keySet(), values); + } +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/TestPipelineBuilder.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/TestPipelineBuilder.java new file mode 100644 index 000000000..624e73a81 --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/TestPipelineBuilder.java @@ -0,0 +1,35 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.flowelements.PipelineBuilderBase; + +/** + * Builds a Pipeline from elements made in code. PipelineBuilder searches + * the class path for element builders each time one is constructed, which + * only a configuration file needs, so the many Pipelines the cases build + * use this instead. + */ +final class TestPipelineBuilder + extends PipelineBuilderBase { +} diff --git a/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/WritesAfterProcessingTests.java b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/WritesAfterProcessingTests.java new file mode 100644 index 000000000..d7d35c2ff --- /dev/null +++ b/pipeline/pipeline.derivedproperty/src/test/java/fiftyone/pipeline/derivedproperty/flowelements/WritesAfterProcessingTests.java @@ -0,0 +1,63 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.derivedproperty.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import org.junit.jupiter.api.Test; + +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * The check, for the Flow Element in this module, the + * {@link DerivedPropertyElement}, that processes it, then writes to its + * Element Data and expects the error. + * @see Specification + */ +public class WritesAfterProcessingTests { + + @Test + public void derivedPropertyElement_ElementDataRefusesWrites() + throws Exception { + DerivedPropertyElement element = DerivedPropertyElementTests.likely(); + try (Pipeline pipeline = DerivedPropertyElementTests.buildWithStubs( + element, DerivedPropertyElementTests.likelyValues()); + FlowData data = pipeline.createFlowData()) { + data.process(); + ElementData own = data.getFromElement(element); + assertThrows( + IllegalStateException.class, + () -> own.put("after", "value"), + "put"); + assertThrows( + IllegalStateException.class, + () -> own.populateFromMap( + Collections.singletonMap( + "after", "value")), + "populateFromMap"); + } + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element-mvc/src/main/java/ExampleLauncher.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element-mvc/src/main/java/ExampleLauncher.java index 3ed7e2bc6..a0f3695ab 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element-mvc/src/main/java/ExampleLauncher.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element-mvc/src/main/java/ExampleLauncher.java @@ -29,6 +29,6 @@ public static void main(String[] args) throws LifecycleException { runWebApp( "pipeline.developer-examples/pipeline.developer-examples.clientside-element-mvc/src/main/webapp", "pipeline.developer-examples/pipeline.developer-examples.clientside-element-mvc/target", - 8088); + 5151); } } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element-mvc/src/main/java/controller/ExampleController.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element-mvc/src/main/java/controller/ExampleController.java index 9294d092d..9c626efbb 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element-mvc/src/main/java/controller/ExampleController.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element-mvc/src/main/java/controller/ExampleController.java @@ -51,6 +51,9 @@ public ExampleController(FlowDataProvider flowDataProvider) { @RequestMapping(method = RequestMethod.GET) public String get(ModelMap model, HttpServletRequest request) { String message = ""; + // A Flow Data cannot be used by two threads at once. Work handed to + // another thread or task gets the values it needs, read here on the + // request's thread, not the Flow Data. try (FlowData data = flowDataProvider.getFlowData(request)) { message = "With a date of birth of " + data.getEvidence().get("cookie.date-of-birth") + diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/pom.xml b/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/pom.xml index a4705ea69..0ece75609 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/pom.xml +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/pom.xml @@ -41,6 +41,11 @@ ${project.version} pipeline.engines + + + org.junit.jupiter + junit-jupiter + diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/src/main/java/fiftyone/pipeline/developerexamples/clientsideelement/data/StarSign.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/src/main/java/fiftyone/pipeline/developerexamples/clientsideelement/data/StarSign.java index 04104cea9..cd5a67ed9 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/src/main/java/fiftyone/pipeline/developerexamples/clientsideelement/data/StarSign.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/src/main/java/fiftyone/pipeline/developerexamples/clientsideelement/data/StarSign.java @@ -32,18 +32,42 @@ public class StarSign { public StarSign(String name, String start, String end) { this.name = name; - this.start = Calendar.getInstance(); - String[] startDate = start.split("/"); - this.start.set( - 0, - Integer.parseInt(startDate[1]) - 1, - Integer.parseInt(startDate[0])); - this.end = Calendar.getInstance(); - String[] endDate = end.split("/"); - this.end.set( + this.start = dayOfYear(start); + this.end = dayOfYear(end); + } + + /** + * The day and month of a "dd/mm" or "dd/mm/yyyy" date, in year 0 at + * midnight, so that two dates compare by day and month alone. + * @param date the date + * @return the day and month + */ + public static Calendar dayOfYear(String date) { + String[] sections = date.split("/"); + Calendar calendar = Calendar.getInstance(); + // Clear the time of day the calendar was made with, which would + // otherwise decide the comparison of two dates on the same day. + calendar.clear(); + calendar.set( 0, - Integer.parseInt(endDate[1]) - 1, - Integer.parseInt(endDate[0])); + Integer.parseInt(sections[1]) - 1, + Integer.parseInt(sections[0])); + return calendar; + } + + /** + * True if the day and month fall within this sign, including its first + * and last days. A sign that runs over the end of the year, as + * Capricorn does, holds the days from its start to the end of December + * and from the start of January to its end. + * @param day a day and month from {@link #dayOfYear(String)} + * @return true if the day falls within this sign + */ + public boolean contains(Calendar day) { + if (start.compareTo(end) <= 0) { + return day.compareTo(start) >= 0 && day.compareTo(end) <= 0; + } + return day.compareTo(start) >= 0 || day.compareTo(end) <= 0; } public String getName() { diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/src/main/java/fiftyone/pipeline/developerexamples/clientsideelement/flowelements/SimpleClientSideElement.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/src/main/java/fiftyone/pipeline/developerexamples/clientsideelement/flowelements/SimpleClientSideElement.java index 86ee6b858..f4d75f647 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/src/main/java/fiftyone/pipeline/developerexamples/clientsideelement/flowelements/SimpleClientSideElement.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/src/main/java/fiftyone/pipeline/developerexamples/clientsideelement/flowelements/SimpleClientSideElement.java @@ -87,16 +87,12 @@ protected void processInternal(FlowData data) throws Exception { boolean validDateOfBirth = false; TryGetResult dateString = data.tryGetEvidence("cookie.date-of-birth", String.class); - Calendar monthAndDay = Calendar.getInstance(); + Calendar monthAndDay = null; if (dateString.hasValue()) { - // "date-of-birth" is there, so parse it. - String[] dateSections = dateString.getValue().split("/"); + // "date-of-birth" is there, so parse its day and month. try { - monthAndDay.set( - 0, - Integer.parseInt(dateSections[1]) - 1, - Integer.parseInt(dateSections[0])); + monthAndDay = StarSign.dayOfYear(dateString.getValue()); validDateOfBirth = true; } catch (Exception e) { @@ -106,8 +102,7 @@ protected void processInternal(FlowData data) throws Exception { if (validDateOfBirth) { // "date-of-birth" is valid, so set the star sign. for (StarSign starSign : starSigns) { - if (monthAndDay.compareTo(starSign.getStart()) >= 0 && - monthAndDay.compareTo(starSign.getEnd()) <= 0) { + if (starSign.contains(monthAndDay)) { elementData.setStarSign(starSign.getName()); break; } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/src/test/java/fiftyone/pipeline/developerexamples/clientsideelement/WritesAfterProcessingTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/src/test/java/fiftyone/pipeline/developerexamples/clientsideelement/WritesAfterProcessingTests.java new file mode 100644 index 000000000..abd38bd4d --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/src/test/java/fiftyone/pipeline/developerexamples/clientsideelement/WritesAfterProcessingTests.java @@ -0,0 +1,87 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.developerexamples.clientsideelement; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.developerexamples.clientsideelement.flowelements.SimpleClientSideElement; +import fiftyone.pipeline.developerexamples.clientsideelement.flowelements.SimpleClientSideElementBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * The check, for the Flow Element in this module, the + * {@link SimpleClientSideElement}, that processes it, then writes to its + * Element Data and expects the error. + * @see Specification + */ +public class WritesAfterProcessingTests { + + private static final ILoggerFactory LOGGER_FACTORY = + LoggerFactory.getILoggerFactory(); + + /** + * Every write method of Element Data fails once its Flow Element has + * finished. + * @param data the Element Data of a finished Flow Element + */ + static void assertWritesFail(ElementData data) { + assertThrows( + IllegalStateException.class, + () -> data.put("after", "value"), + "put"); + assertThrows( + IllegalStateException.class, + () -> data.populateFromMap( + Collections.singletonMap("after", "value")), + "populateFromMap"); + } + + @Test + public void simpleClientSideElement_ElementDataRefusesWrites() + throws Exception { + SimpleClientSideElement element = + new SimpleClientSideElementBuilder(LOGGER_FACTORY).build(); + try (Pipeline pipeline = new PipelineBuilder(LOGGER_FACTORY) + .addFlowElement(element) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("cookie.date-of-birth", "18/12/1992"); + data.process(); + assertEquals(0, data.getErrors().size(), + String.valueOf(data.getErrors())); + + ElementData own = data.getFromElement(element); + assertWritesFail(own); + } + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/src/test/java/pipeline/developerexamples/clientsideelement/ExampleTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/src/test/java/pipeline/developerexamples/clientsideelement/ExampleTests.java new file mode 100644 index 000000000..89bdf3f28 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.clientside-element/src/test/java/pipeline/developerexamples/clientsideelement/ExampleTests.java @@ -0,0 +1,92 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.clientsideelement; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.developerexamples.clientsideelement.data.StarSignData; +import fiftyone.pipeline.developerexamples.clientsideelement.flowelements.SimpleClientSideElement; +import fiftyone.pipeline.developerexamples.clientsideelement.flowelements.SimpleClientSideElementBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The client-side element is one of the core's demonstrations of how to + * create a Flow Element (pipeline-specification/required-examples.md, "The + * examples that MUST be implemented in the core are all demonstrations of + * how to create Flow Elements"). The web example that shows it needs a + * browser, so this test builds the element as that example does, puts it in + * a Pipeline and reads what it gives for each kind of evidence. + */ +public class ExampleTests { + + private static StarSignData process(String dateOfBirth) throws Exception { + SimpleClientSideElement element = + new SimpleClientSideElementBuilder( + LoggerFactory.getILoggerFactory()).build(); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElement(element) + .build(); + FlowData data = pipeline.createFlowData()) { + if (dateOfBirth != null) { + data.addEvidence("cookie.date-of-birth", dateOfBirth); + } + data.process(); + return data.getFromElement(element); + } + } + + @Test + public void NoDateOfBirth_AsksForIt() throws Exception { + StarSignData result = process(null); + assertEquals("Unknown", result.getStarSign()); + assertTrue(result.getDobJavaScript().toString() + .contains("date-of-birth")); + } + + @Test + public void DateOfBirth_GivesStarSign() throws Exception { + StarSignData result = process("01/04/1990"); + assertEquals("Aries", result.getStarSign()); + assertEquals("", result.getDobJavaScript().toString()); + } + + @Test + public void FirstAndLastDaysOfASign_GiveThatSign() throws Exception { + assertEquals("Aries", process("21/03/1990").getStarSign()); + assertEquals("Aries", process("19/04/1990").getStarSign()); + assertEquals("Pisces", process("20/03/1990").getStarSign()); + } + + @Test + public void SignAcrossTheYearEnd_GivesThatSign() throws Exception { + assertEquals("Capricorn", process("22/12/1990").getStarSign()); + assertEquals("Capricorn", process("05/01/1990").getStarSign()); + assertEquals("Capricorn", process("19/01/1990").getStarSign()); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/pom.xml b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/pom.xml index c3dfbc415..46d3d9206 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/pom.xml +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/pom.xml @@ -45,5 +45,9 @@ org.junit.jupiter junit-jupiter + + org.mockito + mockito-core + \ No newline at end of file diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java index 963827bdb..71ec7a741 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java @@ -34,7 +34,7 @@ import pipeline.developerexamples.cloudengine.flowelements.SimpleCloudEngine; import pipeline.developerexamples.cloudengine.flowelements.SimpleCloudEngineBuilder; -import java.util.concurrent.atomic.AtomicBoolean; +import java.io.PrintStream; public class Main { private static final ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory(); @@ -56,24 +56,10 @@ public class Main { "https://cloud.51degrees.com/api/v4/"; /** - * The name the cloud endpoint is read from, and the deprecated name - * read when that one is not set. + * The name the cloud endpoint is read from. */ private static final String ENDPOINT_VARIABLE = "_51DEGREES_CLOUD_ENDPOINT"; - private static final String OLD_ENDPOINT_VARIABLE = "FOD_CLOUD_API_URL"; - - private static final AtomicBoolean oldEndpointVariableLogged = - new AtomicBoolean(false); - - /** - * The deprecated name of the first of the resource key names, read - * when that one is not set. - */ - private static final String OLD_RESOURCE_KEY_VARIABLE = "RESOURCE_KEY"; - - private static final AtomicBoolean oldResourceKeyVariableLogged = - new AtomicBoolean(false); /** * The cloud endpoint, taken from _51DEGREES_CLOUD_ENDPOINT when that is @@ -85,24 +71,10 @@ public class Main { * which is the private hosting option of the cloud service. Both run * the same service, so the example works unchanged. Normalised to * end in one slash so the builder appends its three paths to it - * directly. The deprecated name FOD_CLOUD_API_URL is read when - * _51DEGREES_CLOUD_ENDPOINT is not set, and the first use of it is - * logged as a warning. + * directly. */ static String endpoint() { String value = env(ENDPOINT_VARIABLE); - if (value == null) { - value = env(OLD_ENDPOINT_VARIABLE); - if (value != null - && oldEndpointVariableLogged.compareAndSet(false, true)) { - loggerFactory.getLogger(Main.class.getName()).warn( - "The environment variable " + OLD_ENDPOINT_VARIABLE - + " is deprecated and " + ENDPOINT_VARIABLE - + " replaces it. Set " + ENDPOINT_VARIABLE - + " to the same value, as a later release may no " - + "longer read " + OLD_ENDPOINT_VARIABLE + "."); - } - } if (value == null) { // A resource key and an endpoint travel together, because a // key is only known to the service that issued it. So the @@ -120,40 +92,22 @@ static String endpoint() { } /** - * The names a 51Degrees resource key is read from, in the order they - * are tried. Each is read as an environment variable first and then - * as a system property, so a developer can export one and a build - * can pass one on the command line with -D. + * The name a 51Degrees resource key is read from. It is read as an + * environment variable first and then as a system property, so a + * developer can export one and a build can pass one on the command + * line with -D. */ static final String[] RESOURCE_KEY_NAMES = { "_51DEGREES_RESOURCE_KEY", - "_51DEGREES_RESOURCE_KEY_BESPOKE", - "TestResourceKey", - "SUPER_RESOURCE_KEY", }; /** * The 51Degrees resource key that was found, or null when none of the - * names above is set anywhere. When the first name is not set its - * deprecated name RESOURCE_KEY is read before the others, and the - * first use of it is logged as a warning. + * names above is set anywhere. */ static String configuredResourceKey() { for (String name : RESOURCE_KEY_NAMES) { String value = env(name); - if (value == null && name.equals(RESOURCE_KEY_NAMES[0])) { - value = env(OLD_RESOURCE_KEY_VARIABLE); - if (value != null - && oldResourceKeyVariableLogged.compareAndSet( - false, true)) { - loggerFactory.getLogger(Main.class.getName()).warn( - "The environment variable " + OLD_RESOURCE_KEY_VARIABLE - + " is deprecated and " + name - + " replaces it. Set " + name - + " to the same value, as a later release may no longer " - + "read " + OLD_RESOURCE_KEY_VARIABLE + "."); - } - } if (value != null) { return value; } @@ -233,7 +187,31 @@ public void run() throws Exception { } } + /** + * The rule above and below the "Find out more" section. + */ + private static final String RULE = + "----------------------------------------------------------------"; + + /** + * Ends the output with the pages and repositories that say more about + * what this example shows. + * @param output where to write + */ + static void writeFindOutMore(PrintStream output) { + output.println(); + output.println(RULE); + output.println("Find out more"); + output.println("\tThis example: https://51degrees.com/documentation/_pipeline_api__examples__custom_element__cloud.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.cloud-engine-main.java&utm_term=find-out-more"); + output.println("\tCustom flow elements: https://51degrees.com/documentation/_pipeline_api__features__custom_elements.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.cloud-engine-main.java&utm_term=find-out-more"); + output.println("\thttps://github.com/51Degrees/51degrees-java"); + output.println("\thttps://github.com/51Degrees/specifications"); + output.println(RULE); + output.flush(); + } + public static void main(String[] args) throws Exception { new Example().run(); + writeFindOutMore(System.out); } } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/flowelements/SimpleCloudEngine.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/flowelements/SimpleCloudEngine.java index 89cd49481..f476acb6e 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/flowelements/SimpleCloudEngine.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/flowelements/SimpleCloudEngine.java @@ -34,6 +34,8 @@ import org.json.JSONObject; import org.slf4j.Logger; +import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.NoCloudResponse; + import java.util.ArrayList; import java.util.List; @@ -88,7 +90,24 @@ protected void processEngine(FlowData data, StarSignData aspectData) { } CloudRequestData requestData = data.getFromElement(cloudRequestEngine); - String json = requestData.getJsonResponse(); + // Wait for the cloud response. A failed request is recorded as the + // Cloud Request Engine's error and leaves no response to read. + String json = getJsonResponse( + data, + aspectData, + requestData, + cloudRequestEngine); + if (json == null) { + return; + } + + // A failed cloud request leaves no response. The Cloud Request + // Engine reports why, so this engine adds no error of its own. + if (json == null || json.isEmpty()) { + logger.warn(String.format( + NoCloudResponse, getClass().getSimpleName())); + return; + } // Extract data from json to the aspectData instance. A cloud // service answers only for the products the resource key covers, diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/EndpointTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/EndpointTests.java index c6969a90e..faede0f28 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/EndpointTests.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/EndpointTests.java @@ -39,32 +39,21 @@ public class EndpointTests { private static final String NEW_NAME = "_51DEGREES_CLOUD_ENDPOINT"; - private static final String OLD_NAME = "FOD_CLOUD_API_URL"; @BeforeEach public void setUp() { assumeTrue(System.getenv(NEW_NAME) == null); - assumeTrue(System.getenv(OLD_NAME) == null); clear(); } @AfterEach public void clear() { System.clearProperty(NEW_NAME); - System.clearProperty(OLD_NAME); } @Test - public void Endpoint_Old_Name_Alone() { - System.setProperty(OLD_NAME, "https://old.example/api/v4"); - - assertEquals("https://old.example/api/v4/", Main.endpoint()); - } - - @Test - public void Endpoint_New_Name_Wins() { - System.setProperty(NEW_NAME, "https://new.example/api/v4/"); - System.setProperty(OLD_NAME, "https://old.example/api/v4/"); + public void Endpoint_New_Name() { + System.setProperty(NEW_NAME, "https://new.example/api/v4"); assertEquals("https://new.example/api/v4/", Main.endpoint()); } @@ -79,41 +68,20 @@ public void Endpoint_Neither_Name() { } private static final String NEW_KEY_NAME = "_51DEGREES_RESOURCE_KEY"; - private static final String OLD_KEY_NAME = "RESOURCE_KEY"; - - @Test - public void ResourceKey_Old_Name_Alone() { - assumeTrue(System.getenv(NEW_KEY_NAME) == null); - assumeTrue(System.getenv(OLD_KEY_NAME) == null); - String savedNew = System.clearProperty(NEW_KEY_NAME); - try { - System.setProperty(OLD_KEY_NAME, "old-name-key"); - - assertEquals("old-name-key", Main.configuredResourceKey()); - } finally { - System.clearProperty(OLD_KEY_NAME); - if (savedNew != null) { - System.setProperty(NEW_KEY_NAME, savedNew); - } - } - } @Test - public void ResourceKey_New_Name_Wins() { + public void ResourceKey_New_Name() { assumeTrue(System.getenv(NEW_KEY_NAME) == null); - assumeTrue(System.getenv(OLD_KEY_NAME) == null); String savedNew = System.clearProperty(NEW_KEY_NAME); try { System.setProperty(NEW_KEY_NAME, "new-name-key"); - System.setProperty(OLD_KEY_NAME, "old-name-key"); assertEquals("new-name-key", Main.configuredResourceKey()); } finally { - System.clearProperty(OLD_KEY_NAME); System.clearProperty(NEW_KEY_NAME); if (savedNew != null) { System.setProperty(NEW_KEY_NAME, savedNew); } } } -} +} \ No newline at end of file diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/ExampleTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/ExampleTests.java index 626622038..9285d2640 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/ExampleTests.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/ExampleTests.java @@ -25,7 +25,7 @@ import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.assumeTrue; /** * Runs the cloud engine example against the live cloud service. @@ -35,22 +35,22 @@ * keys, and then processes evidence through it, so any failure to reach * the service, or any answer the pipeline cannot use, has to be seen. * Catching and skipping would report success for a run that never - * reached the service at all. + * reached the service at all. With no resource key the test is skipped, + * naming the variable to set. */ public class ExampleTests { @Test public void SimpleCloudEngine_Test() throws Exception { - if (Main.configuredResourceKey() == null) { - fail("This test runs the cloud engine example against the live " - + "51Degrees cloud service, so it needs a resource key. None " - + "of " + String.join(", ", Main.RESOURCE_KEY_NAMES) + " is " - + "set, as an environment variable or as a system property, " - + "tried in that order. Any key works here, because the " - + "example reports a product the key does not carry rather " - + "than failing on it. Create a key at " - + "https://configure.51degrees.com."); - } + assumeTrue(Main.configuredResourceKey() != null, + "This test runs the cloud engine example against the live " + + "51Degrees cloud service, so it needs a resource key, and " + + "it is skipped because " + + String.join(", ", Main.RESOURCE_KEY_NAMES) + " is not set " + + "as an environment variable or as a system property. Any " + + "key works here, because the example reports a product " + + "the key does not carry rather than failing on it. Create " + + "a key at https://51degrees.com/configure."); new Main.Example().run(); } } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/FindOutMoreTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/FindOutMoreTests.java new file mode 100644 index 000000000..585dd4fa4 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/FindOutMoreTests.java @@ -0,0 +1,103 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.cloudengine; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * The example ends its output with a "Find out more" section, which lists + * pages on 51degrees.com, each with its campaign tags, and the repository. + */ +public class FindOutMoreTests { + + private static final String RULE = + "----------------------------------------------------------------"; + + private static final String HEADING = "Find out more"; + + private static final String REPOSITORY = + "https://github.com/51Degrees/51degrees-java"; + + private static final Pattern TAGGED = Pattern.compile( + "https://51degrees\\.com/[^?\\s]*\\?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=[a-z0-9][a-z0-9._-]*&utm_term=find-out-more$"); + + @Test + public void Main_EndsWithFindOutMore() throws Exception { + assumeTrue(Main.configuredResourceKey() != null, + "This test runs the example against the live cloud service, so " + + "it needs a resource key, and it is skipped because " + + String.join(", ", Main.RESOURCE_KEY_NAMES) + " is not set."); + String output = runMain(); + int example = output.lastIndexOf("star sign"); + int heading = output.lastIndexOf(RULE + "\n" + HEADING + "\n"); + assertTrue(example >= 0, "The example wrote nothing:\n" + output); + assertTrue(heading > example, + "No Find out more section after the example's output:\n" + + output); + String section = output.substring( + heading + RULE.length() + HEADING.length() + 2); + int end = section.indexOf(RULE + "\n"); + assertTrue(end > 0, "The section has no closing rule:\n" + output); + section = section.substring(0, end); + int pages = 0; + for (String line : section.split("\n")) { + if (line.contains("51degrees.com")) { + assertTrue(TAGGED.matcher(line).find(), + "A 51degrees.com link without its campaign tags: " + + line); + pages++; + } + } + assertEquals(2, pages, "The pages listed:\n" + section); + assertTrue(section.contains(REPOSITORY), + "The repository is not listed:\n" + section); + } + + /** + * Runs the example as a person would and returns what it wrote, with + * each line ending in a line feed. + */ + private static String runMain() throws Exception { + PrintStream original = System.out; + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + System.setOut(new PrintStream(captured, true, "UTF-8")); + try { + Main.main(new String[0]); + } finally { + System.setOut(original); + } + return new String(captured.toByteArray(), StandardCharsets.UTF_8) + .replace("\r\n", "\n"); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/WritesAfterProcessingTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/WritesAfterProcessingTests.java new file mode 100644 index 000000000..98d9d8e46 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/WritesAfterProcessingTests.java @@ -0,0 +1,137 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.cloudengine; + +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.TryGetResult; +import fiftyone.pipeline.core.flowelements.FlowElement.DataFactory; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.engines.data.AspectDataBase; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; +import pipeline.developerexamples.cloudengine.flowelements.SimpleCloudEngine; +import pipeline.developerexamples.cloudengine.flowelements.SimpleCloudEngineBuilder; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The check, for the Flow Element in this module, the + * {@link SimpleCloudEngine}, that processes it, then writes to its Element + * Data and expects the error. It is processed through its public process + * method with a fixed cloud response, so the check needs no resource key. + * @see Specification + */ +public class WritesAfterProcessingTests { + + private static final ILoggerFactory LOGGER_FACTORY = + LoggerFactory.getILoggerFactory(); + + /** + * Every write method of Element Data, and of Aspect Data, fails once its + * Flow Element has finished. + * @param data the Element Data of a finished Flow Element + */ + static void assertWritesFail(ElementData data) { + assertThrows( + IllegalStateException.class, + () -> data.put("after", "value"), + "put"); + assertThrows( + IllegalStateException.class, + () -> data.populateFromMap( + Collections.singletonMap("after", "value")), + "populateFromMap"); + if (data instanceof AspectDataBase) { + AspectDataBase aspect = (AspectDataBase) data; + assertThrows( + IllegalStateException.class, + aspect::setCloudRequestFailed, + "setCloudRequestFailed"); + assertThrows( + IllegalStateException.class, + () -> aspect.addEngine(null), + "addEngine"); + assertThrows( + IllegalStateException.class, + () -> aspect.addProcessCallable(null), + "addProcessCallable"); + } + } + + @Test + @SuppressWarnings({"unchecked", "rawtypes"}) + public void simpleCloudEngine_ElementDataRefusesWrites() throws Exception { + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()) + .thenReturn(Collections.emptyMap()); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getProcessStarted()).thenReturn(true); + when(cloudData.getJsonResponse()).thenReturn( + "{ \"starsign\": { \"starsign\": \"Sagittarius\" } }"); + final FlowData flowData = mock(FlowData.class); + final List added = new ArrayList<>(); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + when(flowData.getOrAdd(any(TypedKey.class), any(DataFactory.class))) + .thenAnswer(invocation -> { + if (added.isEmpty()) { + DataFactory factory = invocation.getArgument(1); + added.add(factory.create(flowData)); + } + return added.get(0); + }); + when(flowData.tryGetValue(any(TypedKey.class))) + .thenAnswer(invocation -> { + TryGetResult result = new TryGetResult<>(); + if (added.isEmpty() == false) { + result.setValue(added.get(0)); + } + return result; + }); + + try (SimpleCloudEngine engine = + new SimpleCloudEngineBuilder(LOGGER_FACTORY).build()) { + engine.addPipeline(pipeline); + engine.process(flowData); + + assertEquals(1, added.size()); + assertWritesFail(added.get(0)); + } + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/pom.xml b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/pom.xml new file mode 100644 index 000000000..2847c5aee --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/pom.xml @@ -0,0 +1,50 @@ + + + + + + pipeline.developer-examples + com.51degrees + 4.5.127-SNAPSHOT + + 4.0.0 + + pipeline.developerexamples.derivedproperty + 51Degrees :: Pipeline :: Developer Examples :: Derived Property + https://51degrees.com?utm_source=maven&utm_medium=package&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples-pipeline.developer-examples.derivedproperty-pom.xml&utm_term=url + + + + ${project.groupId} + ${project.version} + pipeline.derivedproperty + + + + org.junit.jupiter + junit-jupiter + + + diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/Main.java new file mode 100644 index 000000000..8246fc9cb --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/Main.java @@ -0,0 +1,374 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.derivedproperty; + +import fiftyone.pipeline.core.configuration.PipelineOptions; +import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.derivedproperty.data.IDerivedPropertyData; +import fiftyone.pipeline.derivedproperty.flowelements.DerivedOutputMetaData; +import fiftyone.pipeline.derivedproperty.flowelements.DerivedPropertyElement; +import fiftyone.pipeline.derivedproperty.flowelements.DerivedPropertyElementBuilder; +import fiftyone.pipeline.derivedproperty.flowelements.DerivedPropertyMetaData; +import fiftyone.pipeline.derivedproperty.flowelements.DerivedValueMetaData; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; +import pipeline.developerexamples.derivedproperty.flowelements.SessionElement; +import pipeline.developerexamples.derivedproperty.flowelements.SessionElementBuilder; + +import java.io.File; +import java.io.PrintStream; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Arrays; +import java.util.List; + +/** + * @example derivedproperty/Main.java + * + * The Derived Property Element computes new properties from properties + * that other elements in the Pipeline have already produced, following a + * script written in YAML or in JSON. The element holds no data file, sends + * no request over the network and needs no resource key, so this example + * runs offline. + * + * The example shows four things. + * + * 1. A script held in the example as text, passed to the builder. + * 2. A script read from a file beside the example, + * src/main/resources/ReaderEngagement.yaml. + * 3. The same Pipeline built from a configuration file, + * src/main/resources/derived-property.xml, naming the builder and + * passing the script through the ScriptFiles build parameter. + * 4. What a caller sees when a source property a script names is not + * available, which is a property with no value and a message naming + * what was missing. + * + * The source properties come from a small element written in this example, + * SessionElement, so that nothing outside is needed. A real Pipeline puts + * device detection or IP intelligence in that position instead. + * + * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/Main.java). + * + * The script file the example reads is + * @include pipeline.developer-examples.derivedproperty/src/main/resources/ReaderEngagement.yaml + * + * and the configuration file is + * @include pipeline.developer-examples.derivedproperty/src/main/resources/derived-property.xml + */ +public class Main { + + private static final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + /** + * The script file beside the example, which both halves of the example + * use, one by naming it on the builder and one by naming it in the + * configuration file. + */ + static final String SCRIPT_FILE = "ReaderEngagement.yaml"; + + /** + * The configuration file beside the example. + */ + static final String CONFIGURATION_FILE = "derived-property.xml"; + + /** + * The property the script in {@link #SCRIPT_FILE} produces. + */ + static final String ENGAGEMENT = "ReaderEngagement"; + + /** + * The property the script held below produces. + */ + static final String EVERY_TEST = "ReaderEngagementIsHigh"; + + /** + * A second script, held as text rather than in a file, to show that a + * script can come straight from your own code. The text is YAML here, + * and JSON is read just as well. It reads the same three source + * properties and says whether every one of the tests passed, so one + * element runs more than one script over one set of source properties. + */ + static final String EVERY_TEST_SCRIPT = String.join("\n", + "Format: 1", + "Name: ReaderEngagementIsHigh", + "Version: \"1.0.0\"", + "Output:", + " Name: ReaderEngagementIsHigh", + " Description: Whether every one of the ReaderEngagement tests " + + "passed on this request.", + " ValueType: bool", + " IsList: false", + " Category: Example", + "Checks:", + " SeveralPages: { Property: session.PagesViewed, Ge: 3 }", + " LongEnough: { Property: session.SecondsSincePageLoad, Ge: 30 }", + " PointerMoved: { Property: session.PointerMoved, Eq: true }", + "Rules:", + " - When: { Failed: Checks, Eq: 0 }", + " Then: true", + " - Else: false", + ""); + + /** + * The rule above and below each part of the output. + */ + private static final String RULE = + "----------------------------------------------------------------"; + + public static class Example { + + /** + * Run the example. + * @param output where to write what the example shows + * @throws Exception where the example cannot run + */ + public void run(PrintStream output) throws Exception { + runFromCode(output); + runFromConfiguration(output); + } + + /** + * The first way to configure the element, being the builder in + * code. One element runs two scripts, one held as text and one read + * from a file. + */ + private void runFromCode(PrintStream output) throws Exception { + output.println(RULE); + output.println("1. Configured with the builder in code"); + output.println(); +//! [usage] + SessionElement session = + new SessionElementBuilder(loggerFactory).build(); + DerivedPropertyElement derived = + new DerivedPropertyElementBuilder(loggerFactory) + // A script from your own environment. A path may hold a + // wildcard, so "scripts/*.yaml" adds a whole folder. + .addScriptFile(resource(SCRIPT_FILE).getPath()) + // A script held as text in your own code. The name + // given here must equal the Name the script carries. + .addScript(EVERY_TEST, EVERY_TEST_SCRIPT) + // addScript(BuiltInScript.HumanConfidence) adds a + // script that ships inside the package. Those scripts + // come from the derived-properties repository, and + // this example uses none of them so that it stands on + // its own. + .build(); + + // The Derived Property Element goes after every element that + // supplies a source property. The Pipeline closes both elements + // when it is closed. + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(session) + .addFlowElement(derived) + .setAutoCloseElements(true) + .build()) { + List properties = Arrays.asList(ENGAGEMENT, EVERY_TEST); + process(output, pipeline, + "Request 1, every source property present.", + 5, 120, true, properties); + process(output, pipeline, + "Request 2, the pointer has not been seen to move.", + 5, 120, null, properties); + } +//! [usage] + output.println("One element runs both scripts, so both " + + "properties are written under the"); + output.println("key derived, and both say the same thing about " + + "the second request because"); + output.println("they read the same source properties. The " + + "second request has no value at"); + output.println("all, because a property the scripts name was " + + "not there, and the message"); + output.println("says which property was missing and what the " + + "element that supplies it said."); + output.println(); + } + + /** + * The second way to configure the element, being a configuration + * file. The file names the builders and the script file, so the + * Pipeline is described entirely outside the code. + */ + private void runFromConfiguration(PrintStream output) + throws Exception { + output.println(RULE); + output.println("2. The same Pipeline built from " + + CONFIGURATION_FILE); + output.println(); + PipelineOptions options = PipelineOptionsFactory + .getOptionsFromFile(resource(CONFIGURATION_FILE)); + // The Pipeline builder makes each element with the builder the + // file names. A script text held in code cannot be given in a + // configuration file, so this Pipeline runs the file's script. + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .buildFromConfiguration(options)) { + List properties = Arrays.asList(ENGAGEMENT); + process(output, pipeline, + "Request 1, every source property present.", + 5, 120, true, properties); + process(output, pipeline, + "Request 2, the pointer has not been seen to move.", + 5, 120, null, properties); + printDefinitions(output, + pipeline.getElement(DerivedPropertyElement.class)); + } + } + } + + /** + * A file beside the example, which the build copies onto the class + * path. + */ + static File resource(String name) throws URISyntaxException { + URL url = Main.class.getClassLoader().getResource(name); + if (url == null) { + throw new IllegalStateException( + "The file " + name + " is not on the class path."); + } + return new File(url.toURI()); + } + + /** + * Process one request and print the source properties and the derived + * properties it produced. + * @param pointerMoved evidence for session.PointerMoved, or null to + * leave the property with no value on this request + */ + private static void process( + PrintStream output, + Pipeline pipeline, + String title, + int pagesViewed, + int seconds, + Boolean pointerMoved, + List derivedProperties) throws Exception { + output.println(title); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(SessionElement.PAGES_VIEWED_EVIDENCE, pagesViewed); + data.addEvidence(SessionElement.SECONDS_EVIDENCE, seconds); + if (pointerMoved != null) { + data.addEvidence( + SessionElement.POINTER_MOVED_EVIDENCE, pointerMoved); + } + data.process(); + + // The source properties, read the way any element's + // properties are read. + ElementData session = data.get(SessionElement.ELEMENT_DATA_KEY); + for (String name : new String[]{SessionElement.PAGES_VIEWED, + SessionElement.SECONDS_SINCE_PAGE_LOAD, + SessionElement.POINTER_MOVED}) { + print(output, "session." + name, + (AspectPropertyValue) session.get(name)); + } + + // The derived properties, read from the Element Data the + // Derived Property Element writes under the key derived. + IDerivedPropertyData derived = data.get(IDerivedPropertyData.class); + for (String name : derivedProperties) { + print(output, "derived." + name, + (AspectPropertyValue) derived.get(name)); + } + } + output.println(); + } + + /** + * Print a value, or, where it has no value, the reason there is none. + */ + private static void print( + PrintStream output, + String name, + AspectPropertyValue value) { + output.println(String.format(" %-33s %s", name, value.hasValue() ? + value.getValue() : + "no value, because " + value.getNoValueMessage())); + } + + /** + * Print the full definition each script carries, read from the element, + * so that everything a script says about its property is available to + * a caller. + */ + private static void printDefinitions( + PrintStream output, + DerivedPropertyElement element) { + for (DerivedPropertyMetaData script : element.getScripts()) { + DerivedOutputMetaData definition = script.getOutput(); + output.println("The definition the script '" + script.getName() + + "' version " + script.getVersion() + " carries."); + output.println(String.format(" %-13s %s", "Name", + definition.getName())); + output.println(String.format(" %-13s %s", "Description", + definition.getDescription())); + output.println(String.format(" %-13s %s", "ValueType", + definition.getValueType())); + output.println(String.format(" %-13s %s", "DefaultValue", + definition.getDefaultValue())); + output.println(String.format(" %-13s %s", "Category", + definition.getCategory())); + output.println(String.format(" %-13s %s", "Dependencies", + definition.getDependencies())); + if (definition.getValues() != null) { + output.println(" Values"); + for (DerivedValueMetaData value : definition.getValues()) { + output.println(String.format(" %-8s %s", + value.getName(), value.getDescription())); + } + } + output.println(); + } + } + + /** + * End the output with the pages and repositories that say more about + * what this example shows. + * @param output where to write + */ + static void writeFindOutMore(PrintStream output) { + output.println(); + output.println(RULE); + output.println("Find out more"); + output.println("\tThe script format, the shared scripts and how to " + + "test a script: https://github.com/51Degrees/derived-properties"); + output.println("\tA page that runs a script in your browser: " + + "https://51degrees.github.io/derived-properties/"); + output.println("\tThe Pipeline API: https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.derivedproperty-main.java&utm_term=find-out-more"); + output.println("\tCustom flow elements, for work a script cannot do: https://51degrees.com/documentation/_pipeline_api__features__custom_elements.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.derivedproperty-main.java&utm_term=find-out-more"); + output.println("\thttps://github.com/51Degrees/51degrees-java"); + output.println("\thttps://github.com/51Degrees/specifications"); + output.println(RULE); + output.flush(); + } + + public static void main(String[] args) throws Exception { + new Example().run(System.out); + writeFindOutMore(System.out); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/flowelements/SessionElement.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/flowelements/SessionElement.java new file mode 100644 index 000000000..3820431fc --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/flowelements/SessionElement.java @@ -0,0 +1,164 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.derivedproperty.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementDataBase; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.ElementPropertyMetaDataDefault; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.TryGetResult; +import fiftyone.pipeline.core.flowelements.FlowElementBase; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; +import org.slf4j.ILoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * Supplies the source properties the example's scripts read. A real + * Pipeline puts device detection or IP intelligence in this position, and + * this small element stands in for one of those so that the example runs + * with no resource key, no data file and no network connection. + *

    + * The element reads three items of evidence and publishes each one under + * the Element Data key {@code session}. Evidence that is not supplied + * leaves the property with no value and a message saying why, which is how + * the example shows a script producing no value and naming what was + * missing. + */ +public final class SessionElement + extends FlowElementBase { + + /** + * The Element Data key, being the part of session.PagesViewed before + * the dot. + */ + public static final String ELEMENT_DATA_KEY = "session"; + + /** + * The evidence holding how many pages the visitor has seen. + */ + public static final String PAGES_VIEWED_EVIDENCE = "query.pagesviewed"; + + /** + * The evidence holding how long the page has been open, in seconds. + */ + public static final String SECONDS_EVIDENCE = "query.secondssincepageload"; + + /** + * The evidence holding whether the pointer has moved. + */ + public static final String POINTER_MOVED_EVIDENCE = "query.pointermoved"; + + public static final String PAGES_VIEWED = "PagesViewed"; + public static final String SECONDS_SINCE_PAGE_LOAD = "SecondsSincePageLoad"; + public static final String POINTER_MOVED = "PointerMoved"; + + private final EvidenceKeyFilter evidenceKeyFilter = + new EvidenceKeyFilterWhitelist(Arrays.asList( + PAGES_VIEWED_EVIDENCE, SECONDS_EVIDENCE, POINTER_MOVED_EVIDENCE)); + + private final List properties; + + /** + * Construct the element. Use {@link SessionElementBuilder} to build one. + * @param loggerFactory the logger factory for the element and its data + */ + SessionElement(ILoggerFactory loggerFactory) { + super(loggerFactory.getLogger(SessionElement.class.getName()), + (flowData, element) -> new ElementDataBase( + loggerFactory.getLogger(ElementDataBase.class.getName()), + flowData) { + }); + // The Derived Property Element checks, when the Pipeline is built, + // that some element lists each source property a script names, so + // these names match the names the scripts use after the dot. + List listed = new ArrayList<>(); + listed.add(new ElementPropertyMetaDataDefault( + PAGES_VIEWED, this, "Example", Integer.class, true)); + listed.add(new ElementPropertyMetaDataDefault( + SECONDS_SINCE_PAGE_LOAD, this, "Example", Integer.class, true)); + listed.add(new ElementPropertyMetaDataDefault( + POINTER_MOVED, this, "Example", Boolean.class, true)); + this.properties = Collections.unmodifiableList(listed); + } + + @Override + public String getElementDataKey() { + return ELEMENT_DATA_KEY; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return evidenceKeyFilter; + } + + @Override + public List getProperties() { + return properties; + } + + @Override + protected void processInternal(FlowData data) { + ElementData session = + data.getOrAdd(getTypedDataKey(), getDataFactory()); + session.put(PAGES_VIEWED, + valueOf(data, PAGES_VIEWED_EVIDENCE, Integer.class)); + session.put(SECONDS_SINCE_PAGE_LOAD, + valueOf(data, SECONDS_EVIDENCE, Integer.class)); + session.put(POINTER_MOVED, + valueOf(data, POINTER_MOVED_EVIDENCE, Boolean.class)); + } + + /** + * The evidence as a property value, or a value with no value saying + * which evidence the request did not carry. + */ + private static AspectPropertyValueDefault valueOf( + FlowData data, + String evidence, + Class type) { + TryGetResult result = data.tryGetEvidence(evidence, type); + if (result.hasValue()) { + return new AspectPropertyValueDefault<>(result.getValue()); + } + AspectPropertyValueDefault missing = + new AspectPropertyValueDefault<>(); + missing.setNoValueMessage( + "the request did not carry the evidence '" + evidence + "'"); + return missing; + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/flowelements/SessionElementBuilder.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/flowelements/SessionElementBuilder.java new file mode 100644 index 000000000..6aebf9a6f --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/java/pipeline/developerexamples/derivedproperty/flowelements/SessionElementBuilder.java @@ -0,0 +1,51 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.derivedproperty.flowelements; + +import fiftyone.pipeline.annotations.ElementBuilder; +import org.slf4j.ILoggerFactory; + +/** + * Builds a {@link SessionElement}. The annotation lets a configuration file + * name the builder as {@code SessionElement}. + */ +@ElementBuilder +public class SessionElementBuilder { + + private final ILoggerFactory loggerFactory; + + /** + * Construct a new builder. + * @param loggerFactory the logger factory for the element + */ + public SessionElementBuilder(ILoggerFactory loggerFactory) { + this.loggerFactory = loggerFactory; + } + + /** + * @return a new element + */ + public SessionElement build() { + return new SessionElement(loggerFactory); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/resources/ReaderEngagement.yaml b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/resources/ReaderEngagement.yaml new file mode 100644 index 000000000..2468a6cd3 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/resources/ReaderEngagement.yaml @@ -0,0 +1,73 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# A derived property script written for this example alone. +# +# The scripts 51Degrees ships inside the pipeline.derivedproperty package +# come from https://github.com/51Degrees/derived-properties and are named +# by BuiltInScript in code, or by the Scripts build parameter in a +# configuration file. This file is not one of them, so the example runs the +# same whatever that repository holds. +# +# The reference for everything below is docs/format-1.md in that +# repository. + +Format: 1 +Name: ReaderEngagement +Version: "1.0.0" +Output: + Name: ReaderEngagement + Description: How far the visitor appears to have engaged with the page, judged only from properties another element has already produced. Returns one of High, Medium or Low. + ValueType: string + DefaultValue: Low + IsList: false + IsMandatory: true + IsObsolete: false + Category: Example + IsPopular: false + ExportValues: true + Values: + - Name: High + Description: Every test went in favour of engagement. + - Name: Medium + Description: One test went against engagement. + - Name: Low + Description: More than one test went against engagement. + +# A property named here is a property the script needs. Where any one of +# the three is missing from a request, the script produces no value and +# says which property was missing and what the element that supplies it +# said, rather than answering on part of the evidence. +Checks: + SeveralPages: { Property: session.PagesViewed, Ge: 3 } + LongEnough: { Property: session.SecondsSincePageLoad, Ge: 30 } + PointerMoved: { Property: session.PointerMoved, Eq: true } + +# Rules are read in order and the first one whose When is true wins. The +# last rule is an Else, which every script ends in, so a script always +# chooses a value once its source properties have been read. +Rules: + - When: { Failed: Checks, Eq: 0 } + Then: High + - When: { Failed: Checks, Eq: 1 } + Then: Medium + - Else: Low diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/resources/derived-property.xml b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/resources/derived-property.xml new file mode 100644 index 000000000..4f1a11883 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/main/resources/derived-property.xml @@ -0,0 +1,57 @@ + + + + + + + + SessionElement + + + DerivedPropertyElement + + + + + ${filepath:ReaderEngagement.yaml} + + + + diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/test/java/pipeline/developerexamples/derivedproperty/ExampleTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/test/java/pipeline/developerexamples/derivedproperty/ExampleTests.java new file mode 100644 index 000000000..a98f07336 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/test/java/pipeline/developerexamples/derivedproperty/ExampleTests.java @@ -0,0 +1,96 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.derivedproperty; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Runs the example and checks it shows the four things the specification + * asks of it, being a script held as text, a script read from a file, the + * same Pipeline built from a configuration file, and a property with no + * value naming the source property that was missing. + */ +public class ExampleTests { + + private static final String NO_VALUE = + "no value, because Derived property '%s' has no value because 1 " + + "source property was not available. 'session.PointerMoved' " + + "(element 'session' has no value for 'PointerMoved': the " + + "request did not carry the evidence 'query.pointermoved')."; + + private static String run() throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (PrintStream output = new PrintStream(bytes, true, "UTF-8")) { + new Main.Example().run(output); + } + return new String(bytes.toByteArray(), StandardCharsets.UTF_8) + .replace("\r\n", "\n"); + } + + private static int count(String text, String part) { + int count = 0; + for (int at = text.indexOf(part); at >= 0; + at = text.indexOf(part, at + 1)) { + count++; + } + return count; + } + + @Test + public void Example_ShowsTheFourThings() throws Exception { + String output = run(); + int code = output.indexOf("1. Configured with the builder in code"); + int file = output.indexOf("2. The same Pipeline built from " + + "derived-property.xml"); + assertTrue(code >= 0 && file > code, output); + String fromCode = output.substring(code, file); + String fromFile = output.substring(file); + + // The script held as text and the script read from a file each + // give a value from the source properties. + assertTrue(fromCode.contains( + "derived.ReaderEngagementIsHigh true"), fromCode); + assertTrue(fromCode.contains( + "derived.ReaderEngagement High"), fromCode); + // The configuration file builds the same Pipeline with the file. + assertTrue(fromFile.contains( + "derived.ReaderEngagement High"), fromFile); + assertTrue(fromFile.contains("The definition the script " + + "'ReaderEngagement' version 1.0.0 carries."), fromFile); + // A source property that is not available leaves no value, and + // the message names it. + assertEquals(1, count(fromCode, + String.format(NO_VALUE, "ReaderEngagementIsHigh")), fromCode); + assertEquals(1, count(fromCode, + String.format(NO_VALUE, "ReaderEngagement")), fromCode); + assertEquals(1, count(fromFile, + String.format(NO_VALUE, "ReaderEngagement")), fromFile); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/test/java/pipeline/developerexamples/derivedproperty/FindOutMoreTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/test/java/pipeline/developerexamples/derivedproperty/FindOutMoreTests.java new file mode 100644 index 000000000..9a7b110bb --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/test/java/pipeline/developerexamples/derivedproperty/FindOutMoreTests.java @@ -0,0 +1,103 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.derivedproperty; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The example ends its output with a "Find out more" section, which lists + * pages on 51degrees.com, each with its campaign tags, and the + * repositories. + */ +public class FindOutMoreTests { + + private static final String RULE = + "----------------------------------------------------------------"; + + private static final String HEADING = "Find out more"; + + private static final String[] REPOSITORIES = { + "https://github.com/51Degrees/51degrees-java", + "https://github.com/51Degrees/derived-properties", + "https://github.com/51Degrees/specifications"}; + + private static final Pattern TAGGED = Pattern.compile( + "https://51degrees\\.com/[^?\\s]*\\?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=[a-z0-9][a-z0-9._-]*&utm_term=find-out-more$"); + + @Test + public void Main_EndsWithFindOutMore() throws Exception { + String output = runMain(); + int example = output.lastIndexOf("derived.ReaderEngagement"); + int heading = output.lastIndexOf(RULE + "\n" + HEADING + "\n"); + assertTrue(example >= 0, "The example wrote nothing:\n" + output); + assertTrue(heading > example, + "No Find out more section after the example's output:\n" + + output); + String section = output.substring( + heading + RULE.length() + HEADING.length() + 2); + int end = section.indexOf(RULE + "\n"); + assertTrue(end > 0, "The section has no closing rule:\n" + output); + section = section.substring(0, end); + int pages = 0; + for (String line : section.split("\n")) { + if (line.contains("51degrees.com")) { + assertTrue(TAGGED.matcher(line).find(), + "A 51degrees.com link without its campaign tags: " + + line); + pages++; + } + } + assertEquals(2, pages, "The pages listed:\n" + section); + for (String repository : REPOSITORIES) { + assertTrue(section.contains(repository), + repository + " is not listed:\n" + section); + } + } + + /** + * Runs the example as a person would and returns what it wrote, with + * each line ending in a line feed. + */ + private static String runMain() throws Exception { + PrintStream original = System.out; + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + System.setOut(new PrintStream(captured, true, "UTF-8")); + try { + Main.main(new String[0]); + } finally { + System.setOut(original); + } + return new String(captured.toByteArray(), StandardCharsets.UTF_8) + .replace("\r\n", "\n"); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/test/java/pipeline/developerexamples/derivedproperty/WritesAfterProcessingTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/test/java/pipeline/developerexamples/derivedproperty/WritesAfterProcessingTests.java new file mode 100644 index 000000000..f16c54fb7 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.derivedproperty/src/test/java/pipeline/developerexamples/derivedproperty/WritesAfterProcessingTests.java @@ -0,0 +1,73 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.derivedproperty; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; +import pipeline.developerexamples.derivedproperty.flowelements.SessionElement; +import pipeline.developerexamples.derivedproperty.flowelements.SessionElementBuilder; + +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * The check, for the Flow Element in this module, the + * {@link SessionElement}, that processes it, then writes to its Element + * Data and expects the error. + * @see Specification + */ +public class WritesAfterProcessingTests { + + private static final ILoggerFactory LOGGER_FACTORY = + LoggerFactory.getILoggerFactory(); + + @Test + public void sessionElement_ElementDataRefusesWrites() throws Exception { + SessionElement element = + new SessionElementBuilder(LOGGER_FACTORY).build(); + try (Pipeline pipeline = new PipelineBuilder(LOGGER_FACTORY) + .addFlowElement(element) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + data.process(); + ElementData own = data.getFromElement(element); + assertThrows( + IllegalStateException.class, + () -> own.put("after", "value"), + "put"); + assertThrows( + IllegalStateException.class, + () -> own.populateFromMap( + Collections.singletonMap( + "after", "value")), + "populateFromMap"); + } + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/main/java/pipeline/developerexamples/flowelement/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/main/java/pipeline/developerexamples/flowelement/Main.java index 76d7e4473..2fe22edcf 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/main/java/pipeline/developerexamples/flowelement/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/main/java/pipeline/developerexamples/flowelement/Main.java @@ -30,6 +30,7 @@ import pipeline.developerexamples.flowelement.flowelements.SimpleFlowElement; import pipeline.developerexamples.flowelement.flowelements.SimpleFlowElementBuilder; +import java.io.PrintStream; import java.text.SimpleDateFormat; import java.util.Calendar; @@ -69,7 +70,31 @@ public void run() throws Exception { } } + /** + * The rule above and below the "Find out more" section. + */ + private static final String RULE = + "----------------------------------------------------------------"; + + /** + * Ends the output with the pages and repositories that say more about + * what this example shows. + * @param output where to write + */ + static void writeFindOutMore(PrintStream output) { + output.println(); + output.println(RULE); + output.println("Find out more"); + output.println("\tThis example: https://51degrees.com/documentation/_pipeline_api__examples__custom_element__flow_element.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.flowelement-main.java&utm_term=find-out-more"); + output.println("\tCustom flow elements: https://51degrees.com/documentation/_pipeline_api__features__custom_elements.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.flowelement-main.java&utm_term=find-out-more"); + output.println("\thttps://github.com/51Degrees/51degrees-java"); + output.println("\thttps://github.com/51Degrees/specifications"); + output.println(RULE); + output.flush(); + } + public static void main(String[] args) throws Exception { new Example().run(); + writeFindOutMore(System.out); } } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/test/java/pipeline/developerexamples/flowelement/FindOutMoreTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/test/java/pipeline/developerexamples/flowelement/FindOutMoreTests.java new file mode 100644 index 000000000..b180ac756 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/test/java/pipeline/developerexamples/flowelement/FindOutMoreTests.java @@ -0,0 +1,98 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.flowelement; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The example ends its output with a "Find out more" section, which lists + * pages on 51degrees.com, each with its campaign tags, and the repository. + */ +public class FindOutMoreTests { + + private static final String RULE = + "----------------------------------------------------------------"; + + private static final String HEADING = "Find out more"; + + private static final String REPOSITORY = + "https://github.com/51Degrees/51degrees-java"; + + private static final Pattern TAGGED = Pattern.compile( + "https://51degrees\\.com/[^?\\s]*\\?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=[a-z0-9][a-z0-9._-]*&utm_term=find-out-more$"); + + @Test + public void Main_EndsWithFindOutMore() throws Exception { + String output = runMain(); + int example = output.lastIndexOf("your star sign is"); + int heading = output.lastIndexOf(RULE + "\n" + HEADING + "\n"); + assertTrue(example >= 0, "The example wrote nothing:\n" + output); + assertTrue(heading > example, + "No Find out more section after the example's output:\n" + + output); + String section = output.substring( + heading + RULE.length() + HEADING.length() + 2); + int end = section.indexOf(RULE + "\n"); + assertTrue(end > 0, "The section has no closing rule:\n" + output); + section = section.substring(0, end); + int pages = 0; + for (String line : section.split("\n")) { + if (line.contains("51degrees.com")) { + assertTrue(TAGGED.matcher(line).find(), + "A 51degrees.com link without its campaign tags: " + + line); + pages++; + } + } + assertEquals(2, pages, "The pages listed:\n" + section); + assertTrue(section.contains(REPOSITORY), + "The repository is not listed:\n" + section); + } + + /** + * Runs the example as a person would and returns what it wrote, with + * each line ending in a line feed. + */ + private static String runMain() throws Exception { + PrintStream original = System.out; + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + System.setOut(new PrintStream(captured, true, "UTF-8")); + try { + Main.main(new String[0]); + } finally { + System.setOut(original); + } + return new String(captured.toByteArray(), StandardCharsets.UTF_8) + .replace("\r\n", "\n"); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/test/java/pipeline/developerexamples/flowelement/WritesAfterProcessingTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/test/java/pipeline/developerexamples/flowelement/WritesAfterProcessingTests.java new file mode 100644 index 000000000..3e1293d4e --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.flowelement/src/test/java/pipeline/developerexamples/flowelement/WritesAfterProcessingTests.java @@ -0,0 +1,89 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.flowelement; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; +import pipeline.developerexamples.flowelement.flowelements.SimpleFlowElement; +import pipeline.developerexamples.flowelement.flowelements.SimpleFlowElementBuilder; + +import java.util.Calendar; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * The check, for the Flow Element in this module, the + * {@link SimpleFlowElement}, that processes it, then writes to its Element + * Data and expects the error. + * @see Specification + */ +public class WritesAfterProcessingTests { + + private static final ILoggerFactory LOGGER_FACTORY = + LoggerFactory.getILoggerFactory(); + + /** + * Every write method of Element Data fails once its Flow Element has + * finished. + * @param data the Element Data of a finished Flow Element + */ + static void assertWritesFail(ElementData data) { + assertThrows( + IllegalStateException.class, + () -> data.put("after", "value"), + "put"); + assertThrows( + IllegalStateException.class, + () -> data.populateFromMap( + Collections.singletonMap("after", "value")), + "populateFromMap"); + } + + @Test + public void simpleFlowElement_ElementDataRefusesWrites() throws Exception { + SimpleFlowElement element = + new SimpleFlowElementBuilder(LOGGER_FACTORY).build(); + Calendar dob = Calendar.getInstance(); + dob.set(1992, Calendar.DECEMBER, 18); + try (Pipeline pipeline = new PipelineBuilder(LOGGER_FACTORY) + .addFlowElement(element) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("date-of-birth", dob.getTime()); + data.process(); + assertEquals(0, data.getErrors().size(), + String.valueOf(data.getErrors())); + + ElementData own = data.getFromElement(element); + assertWritesFail(own); + } + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/README.md b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/README.md index 73ab80af9..d1019caab 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/README.md +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/README.md @@ -93,7 +93,7 @@ scope. | `_51DEGREES_RESOURCE_KEY` | Required. The resource key of the page, public by nature. | | `_51DEGREES_LICENSE_KEY` | Optional. A licence key of the same account, server side only. Only an account that holds licence keys needs one to redeem, because the licence key is what keeps redemption to the acting party's own servers, so an account holding none redeems without one. | | `_51DEGREES_CLOUD_ENDPOINT` | Optional. The cloud API base including the `/api/v4/` segment, defaulting to `https://cloud.51degrees.com/api/v4/`. This is the same variable the cloud request engine in this repository honours, so setting it once points every 51Degrees example at the same place. A trailing slash is added where missing. A host other than cloud.51degrees.com would be used to (a) use an on premise web server, or (b) use a privately hosted version of the 51Degrees cloud for performance reasons, which is the private hosting option of the cloud service. Both run the same service, so the examples work unchanged. | -| `PORT` | The port to listen on, defaulting to `5100`. | +| `PORT` | The port to listen on, defaulting to `5151`. | ### How to run @@ -108,7 +108,7 @@ mvn -pl pipeline.developer-examples/pipeline.developer-examples.fodid exec:java The module depends on `pipeline.did` from this repository, so the first command builds that package too. -The demo server prints the address to open, `http://localhost:5100/` +The demo server prints the address to open, `http://localhost:5151/` by default. A creator context verdict of `nocontext` is a normal outcome rather than an error, because a self-hosted container may be configured not to emit the creator context, so an identifier it issued diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java index 3eaf3c223..e22378f89 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java @@ -102,28 +102,30 @@ * variables: *

      *
    • {@code _51DEGREES_RESOURCE_KEY}, required. The resource key of the - * page, public by nature. Its deprecated name {@code RESOURCE_KEY} is read - * when it is not set.
    • + * page, public by nature. *
    • {@code _51DEGREES_LICENSE_KEY}, optional. A licence key of the same - * account, server side only. Its deprecated name {@code LICENSE_KEY} is - * read when it is not set.
    • + * account, server side only. *
    • {@code _51DEGREES_CLOUD_ENDPOINT}, optional. The cloud API base including * the {@code /api/v4/} segment, defaulting to * {@code https://cloud.51degrees.com/api/v4/}. This is the same variable * the cloud request engine of this package honours.
    • - *
    • {@code PORT}, optional, defaulting to {@code 5100}.
    • + *
    • {@code PORT}, optional, defaulting to {@code 5151}.
    • *
    - * Then open {@code http://localhost:5100/}. + * Then open {@code http://localhost:5151/}. */ public class CreatorContextDemoServer { /** Where the page and stylesheet live on the classpath. */ static final String RESOURCES = "/fodid/creator-context/"; - static final String RESOURCE = - key("_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY"); - static final String LICENCE = - key("_51DEGREES_LICENSE_KEY", "LICENSE_KEY"); + /** + * The port the server listens on when the PORT environment variable + * does not choose another. + */ + public static final int DEFAULT_PORT = 5151; + + static final String RESOURCE = env("_51DEGREES_RESOURCE_KEY"); + static final String LICENCE = env("_51DEGREES_LICENSE_KEY"); /** * The one client, built at start-up and shared by every request, @@ -133,9 +135,15 @@ public class CreatorContextDemoServer { public static void main(String[] args) throws Exception { if (RESOURCE == null) { - System.err.println( - "Set _51DEGREES_RESOURCE_KEY to the resource key of the page."); - System.exit(1); + System.out.println("This demo calls the 51Degrees cloud " + + "service, which needs a resource key, and none is set."); + System.out.println("Set the _51DEGREES_RESOURCE_KEY environment " + + "variable to the resource key of the page, and run the demo " + + "again."); + System.out.println("Create a resource key at " + + "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.fodid-creatorcontextdemoserver.java&utm_term=no-resource-key"); + Main.writeFindOutMore(System.out); + return; } if (LICENCE == null) { // Only an account that holds licence keys needs one to @@ -165,7 +173,7 @@ public static void main(String[] args) throws Exception { // web examples vendor it. resource("page.html"); resource("examples-main.min.css"); - int port = Integer.parseInt(envOr("PORT", "5100")); + int port = Integer.parseInt(envOr("PORT", String.valueOf(DEFAULT_PORT))); HttpServer server = HttpServer.create( new InetSocketAddress(port), 0); server.createContext("/", CreatorContextDemoServer::servePage); @@ -464,27 +472,6 @@ static String env(String... names) { return null; } - /** - * The value of the named environment variable, or of its deprecated - * old name when the named one is not set, saying on standard error - * that the old name is deprecated when it is the one used. - */ - static String key(String name, String oldName) { - String value = env(name); - if (value == null) { - value = env(oldName); - if (value != null) { - System.err.println( - "The environment variable " + oldName - + " is deprecated and " + name - + " replaces it. Set " + name - + " to the same value, as a later release may no longer " - + "read " + oldName + "."); - } - } - return value; - } - static String envOr(String name, String fallback) { String value = env(name); return value == null ? fallback : value; diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/Main.java index d21426abc..26d0f0b61 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/Main.java @@ -26,6 +26,7 @@ import com.swancommunity.owid.Crypto; import fiftyone.pipeline.did.FodId; +import java.io.PrintStream; import java.util.Arrays; /** @@ -140,7 +141,31 @@ private static String toHex(byte[] bytes) { } } + /** + * The rule above and below the "Find out more" section. + */ + private static final String RULE = + "----------------------------------------------------------------"; + + /** + * Ends the output with the pages and repositories that say more about + * what this example shows. + * @param output where to write + */ + static void writeFindOutMore(PrintStream output) { + output.println(); + output.println(RULE); + output.println("Find out more"); + output.println("\t51Did: https://51degrees.com/documentation/_identifiers__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.fodid-main.java&utm_term=find-out-more"); + output.println("\tAsk about 51Did: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.fodid-main.java&utm_term=find-out-more"); + output.println("\thttps://github.com/51Degrees/51degrees-java"); + output.println("\thttps://github.com/51Degrees/specifications"); + output.println(RULE); + output.flush(); + } + public static void main(String[] args) throws Exception { new Example().run(); + writeFindOutMore(System.out); } } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/KeyProbe.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/CreatorContextDemoServerPortTests.java similarity index 62% rename from pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/KeyProbe.java rename to pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/CreatorContextDemoServerPortTests.java index 074504798..37c94f6d5 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/KeyProbe.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/CreatorContextDemoServerPortTests.java @@ -22,27 +22,19 @@ package pipeline.developerexamples.fodid; -/** - * Prints the resource key and the licence key the example read from the - * environment of the process. {@link DeprecatedKeyNameTests} runs this in - * a Java process of its own, because the example reads both keys once, - * when its class loads, and a process cannot change its own environment - * variables. - */ -public final class KeyProbe { +import org.junit.jupiter.api.Test; - static final String RESOURCE_PREFIX = "RESOURCE "; - static final String LICENCE_PREFIX = "LICENCE "; +import static org.junit.jupiter.api.Assertions.assertEquals; - private KeyProbe() { - } +/** + * Checks that the 51Did creator context demo server listens on port 5151 + * when the PORT environment variable does not choose another, as the + * readme says. + */ +class CreatorContextDemoServerPortTests { - public static void main(String[] args) { - System.out.println( - RESOURCE_PREFIX + CreatorContextDemoServer.RESOURCE); - System.out.println( - LICENCE_PREFIX + CreatorContextDemoServer.LICENCE); - System.out.flush(); - System.exit(0); + @Test + void listensOn5151ByDefault() { + assertEquals(5151, CreatorContextDemoServer.DEFAULT_PORT); } } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/CreatorContextDemoServerWithoutKeyTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/CreatorContextDemoServerWithoutKeyTests.java new file mode 100644 index 000000000..c8da2fb1e --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/CreatorContextDemoServerWithoutKeyTests.java @@ -0,0 +1,66 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.fodid; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Runs the 51Did creator context demo server as it runs from a fresh clone, + * with no resource key anywhere, and checks that it ends without an error, + * names the variable to set and links to where a key is created, then ends + * with its Find out more section. + */ +class CreatorContextDemoServerWithoutKeyTests { + + private static final String CONFIGURE = "https://51degrees.com/configure"; + + private static final String TAGS = + "?utm_source=code&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content="; + + private static void runWithoutKey(Class example) throws Exception { + ExampleProcess.Result result = ExampleProcess.runWithoutKeys(example); + System.out.println(result.output); + String output = result.output; + assertEquals(0, result.exitCode, + example.getSimpleName() + " should end with exit code 0"); + assertTrue(output.contains( + "Set the _51DEGREES_RESOURCE_KEY environment variable"), + "The output should name _51DEGREES_RESOURCE_KEY"); + int link = output.indexOf(CONFIGURE); + assertTrue(link >= 0 && output.startsWith(TAGS, + output.indexOf('?', link)), + "The output should link to where a key is created"); + assertTrue(output.lastIndexOf("Find out more") > link, + "The output should end with the Find out more " + + "section after the link"); + } + + @Test + void creatorContextDemoServer() throws Exception { + runWithoutKey(CreatorContextDemoServer.class); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/DeprecatedKeyNameTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/DeprecatedKeyNameTests.java deleted file mode 100644 index 9aab1e55c..000000000 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/DeprecatedKeyNameTests.java +++ /dev/null @@ -1,196 +0,0 @@ -/* ********************************************************************* - * This Original Work is copyright of 51 Degrees Mobile Experts Limited. - * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, - * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. - * - * This Original Work is licensed under the European Union Public Licence - * (EUPL) v.1.2 and is subject to its terms as set out below. - * - * If a copy of the EUPL was not distributed with this file, You can obtain - * one at https://opensource.org/licenses/EUPL-1.2. - * - * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be - * amended by the European Commission) shall be deemed incompatible for - * the purposes of the Work and the provisions of the compatibility - * clause in Article 5 of the EUPL shall not apply. - * - * If using the Work as, or as part of, a network application, by - * including the attribution notice(s) required under Article 5 of the EUPL - * in the end user terms of the application under an appropriate heading, - * such notice(s) shall fulfill the requirements of that article. - * ********************************************************************* */ - -package pipeline.developerexamples.fodid; - -import fiftyone.pipeline.did.DidClient; -import org.json.JSONObject; -import org.junit.jupiter.api.Test; - -import java.io.BufferedReader; -import java.io.File; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Tests which environment variables the example takes its resource key and - * its licence key from. Each test starts {@link KeyProbe} in a Java process - * of its own with the variables it needs, so the example reads a real - * environment. - */ -public class DeprecatedKeyNameTests { - - private static final String RESOURCE_KEY_NAME = "_51DEGREES_RESOURCE_KEY"; - private static final String OLD_RESOURCE_KEY_NAME = "RESOURCE_KEY"; - private static final String LICENCE_KEY_NAME = "_51DEGREES_LICENSE_KEY"; - private static final String OLD_LICENCE_KEY_NAME = "LICENSE_KEY"; - - private static final int PROBE_TIMEOUT_SECONDS = 120; - - @Test - public void oldNamesAlone_GiveTheirKeys() throws Exception { - List output = - probe(null, "old-name-key", null, "old-name-licence"); - - assertEquals("old-name-key", - value(output, KeyProbe.RESOURCE_PREFIX), output.toString()); - assertEquals("old-name-licence", - value(output, KeyProbe.LICENCE_PREFIX), output.toString()); - } - - @Test - public void oldNamesAlone_AreEachReportedAsDeprecatedOnce() - throws Exception { - List output = - probe(null, "old-name-key", null, "old-name-licence"); - - assertEquals(1, deprecations(output, OLD_RESOURCE_KEY_NAME, - RESOURCE_KEY_NAME), output.toString()); - assertEquals(1, deprecations(output, OLD_LICENCE_KEY_NAME, - LICENCE_KEY_NAME), output.toString()); - } - - @Test - public void bothNames_NewNamesWin() throws Exception { - List output = probe( - "new-name-key", "old-name-key", - "new-name-licence", "old-name-licence"); - - assertEquals("new-name-key", - value(output, KeyProbe.RESOURCE_PREFIX), output.toString()); - assertEquals("new-name-licence", - value(output, KeyProbe.LICENCE_PREFIX), output.toString()); - assertEquals(0, deprecations(output, OLD_RESOURCE_KEY_NAME, - RESOURCE_KEY_NAME), output.toString()); - assertEquals(0, deprecations(output, OLD_LICENCE_KEY_NAME, - LICENCE_KEY_NAME), output.toString()); - } - - @Test - public void noNames_GiveNoKeys() throws Exception { - List output = probe(null, null, null, null); - - assertEquals("null", - value(output, KeyProbe.RESOURCE_PREFIX), output.toString()); - assertEquals("null", - value(output, KeyProbe.LICENCE_PREFIX), output.toString()); - } - - /** - * Runs the probe with each variable set to the value given, or absent - * where the value is null, and returns the lines it printed. - */ - private static List probe( - String resourceKey, - String oldResourceKey, - String licenceKey, - String oldLicenceKey) throws Exception { - String java = Paths.get( - System.getProperty("java.home"), "bin", "java").toString(); - ProcessBuilder builder = - new ProcessBuilder(java, KeyProbe.class.getName()); - Map environment = builder.environment(); - environment.put("CLASSPATH", classPath()); - set(environment, RESOURCE_KEY_NAME, resourceKey); - set(environment, OLD_RESOURCE_KEY_NAME, oldResourceKey); - set(environment, LICENCE_KEY_NAME, licenceKey); - set(environment, OLD_LICENCE_KEY_NAME, oldLicenceKey); - builder.redirectErrorStream(true); - - Process process = builder.start(); - List output = new ArrayList<>(); - try (BufferedReader reader = new BufferedReader(new InputStreamReader( - process.getInputStream(), StandardCharsets.UTF_8))) { - String line; - while ((line = reader.readLine()) != null) { - output.add(line); - } - } - assertTrue( - process.waitFor(PROBE_TIMEOUT_SECONDS, TimeUnit.SECONDS), - "The probe did not finish."); - assertEquals(0, process.exitValue(), output.toString()); - return output; - } - - private static void set( - Map environment, String name, String value) { - if (value == null) { - environment.remove(name); - } else { - environment.put(name, value); - } - } - - /** - * The places the probe loads its classes from, being the classes of - * the example, its test classes, the 51Did package and the JSON - * library. - */ - private static String classPath() throws Exception { - return location(CreatorContextDemoServer.class) + File.pathSeparator - + location(KeyProbe.class) + File.pathSeparator - + location(DidClient.class) + File.pathSeparator - + location(JSONObject.class); - } - - private static String location(Class type) throws Exception { - return Paths.get(type.getProtectionDomain().getCodeSource() - .getLocation().toURI()).toString(); - } - - /** The text after the prefix on the one line that starts with it. */ - private static String value(List output, String prefix) { - String found = null; - for (String line : output) { - if (line.startsWith(prefix)) { - assertNull(found, output.toString()); - found = line.substring(prefix.length()); - } - } - return found; - } - - /** - * How many lines say the old name is deprecated and name the new one. - */ - private static int deprecations( - List output, String oldName, String newName) { - int count = 0; - for (String line : output) { - if (line.contains(" " + oldName + " is deprecated") - && line.contains(newName)) { - count++; - } - } - return count; - } -} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/ExampleProcess.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/ExampleProcess.java new file mode 100644 index 000000000..330f1e554 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/ExampleProcess.java @@ -0,0 +1,127 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.fodid; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +/** + * Runs an example's main method in a new Java process with no key in its + * environment, as someone runs it from a fresh clone, for the tests of + * what an example does with no settings. A new process is needed because + * a running Java process cannot remove a variable from its own + * environment. + */ +final class ExampleProcess { + + /** + * How long an example may run before the test stops it. + */ + private static final long TIMEOUT_SECONDS = 120; + + /** + * What an example wrote and how it ended. + */ + public static final class Result { + /** + * The process's exit code. + */ + public final int exitCode; + + /** + * Everything the process wrote to standard output and standard + * error. + */ + public final String output; + + Result(int exitCode, String output) { + this.exitCode = exitCode; + this.output = output; + } + } + + private ExampleProcess() { + } + + /** + * Runs the example's main method with no arguments and with every + * environment variable whose name holds "KEY", in any case, left out. + * The process takes the class path of the process that starts it, and + * of its system properties only project.root, which says where the + * files the example reads are. + * @param example the class whose main method runs + * @return the exit code and the output + * @throws Exception if the process cannot start, or does not end + * within the timeout + */ + public static Result runWithoutKeys(Class example) throws Exception { + List command = new ArrayList<>(); + command.add(new File(new File(System.getProperty("java.home"), + "bin"), "java").getPath()); + command.add("-cp"); + command.add(System.getProperty("java.class.path")); + String projectRoot = System.getProperty("project.root"); + if (projectRoot != null) { + command.add("-Dproject.root=" + projectRoot); + } + command.add(example.getName()); + + ProcessBuilder builder = new ProcessBuilder(command); + builder.environment().keySet().removeIf( + name -> name.toUpperCase(Locale.ROOT).contains("KEY")); + builder.redirectErrorStream(true); + Process process = builder.start(); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + Thread reader = new Thread(() -> copy(process.getInputStream(), + output)); + reader.start(); + if (process.waitFor(TIMEOUT_SECONDS, TimeUnit.SECONDS) == false) { + process.destroyForcibly(); + reader.join(); + throw new AssertionError(example.getName() + + " did not end within " + TIMEOUT_SECONDS + " seconds. " + + "It wrote:\n" + output.toString("UTF-8")); + } + reader.join(); + return new Result(process.exitValue(), output.toString("UTF-8")); + } + + private static void copy(InputStream input, ByteArrayOutputStream output) { + byte[] buffer = new byte[8192]; + try { + int read; + while ((read = input.read(buffer)) >= 0) { + output.write(buffer, 0, read); + } + } catch (IOException e) { + // The process ended, so there is no more to read. + } + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/FindOutMoreTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/FindOutMoreTests.java new file mode 100644 index 000000000..e34a4bd11 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/FindOutMoreTests.java @@ -0,0 +1,98 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.fodid; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The example ends its output with a "Find out more" section, which lists + * pages on 51degrees.com, each with its campaign tags, and the repository. + */ +public class FindOutMoreTests { + + private static final String RULE = + "----------------------------------------------------------------"; + + private static final String HEADING = "Find out more"; + + private static final String REPOSITORY = + "https://github.com/51Degrees/51degrees-java"; + + private static final Pattern TAGGED = Pattern.compile( + "https://51degrees\\.com/[^?\\s]*\\?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=[a-z0-9][a-z0-9._-]*&utm_term=find-out-more$"); + + @Test + public void Main_EndsWithFindOutMore() throws Exception { + String output = runMain(); + int example = output.lastIndexOf("Same match key"); + int heading = output.lastIndexOf(RULE + "\n" + HEADING + "\n"); + assertTrue(example >= 0, "The example wrote nothing:\n" + output); + assertTrue(heading > example, + "No Find out more section after the example's output:\n" + + output); + String section = output.substring( + heading + RULE.length() + HEADING.length() + 2); + int end = section.indexOf(RULE + "\n"); + assertTrue(end > 0, "The section has no closing rule:\n" + output); + section = section.substring(0, end); + int pages = 0; + for (String line : section.split("\n")) { + if (line.contains("51degrees.com")) { + assertTrue(TAGGED.matcher(line).find(), + "A 51degrees.com link without its campaign tags: " + + line); + pages++; + } + } + assertEquals(2, pages, "The pages listed:\n" + section); + assertTrue(section.contains(REPOSITORY), + "The repository is not listed:\n" + section); + } + + /** + * Runs the example as a person would and returns what it wrote, with + * each line ending in a line feed. + */ + private static String runMain() throws Exception { + PrintStream original = System.out; + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + System.setOut(new PrintStream(captured, true, "UTF-8")); + try { + Main.main(new String[0]); + } finally { + System.setOut(original); + } + return new String(captured.toByteArray(), StandardCharsets.UTF_8) + .replace("\r\n", "\n"); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/resources/fodid/creator-context/creator-context-page-harness.js b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/resources/fodid/creator-context/creator-context-page-harness.js index 44c4bcaf3..fa51d4ac6 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/resources/fodid/creator-context/creator-context-page-harness.js +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/resources/fodid/creator-context/creator-context-page-harness.js @@ -151,7 +151,7 @@ const sandbox = { navigator: { clipboard: { writeText: function () {} } }, location: { search: given ? '?51did=' + encodeURIComponent(given) : '', - origin: 'http://localhost:5100', + origin: 'http://localhost:5151', pathname: '/' } }; diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/main/java/pipeline/developerexamples/onpremiseengine/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/main/java/pipeline/developerexamples/onpremiseengine/Main.java index 8d803d244..0d585136c 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/main/java/pipeline/developerexamples/onpremiseengine/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/main/java/pipeline/developerexamples/onpremiseengine/Main.java @@ -30,6 +30,7 @@ import pipeline.developerexamples.onpremiseengine.flowelements.SimpleOnPremiseEngine; import pipeline.developerexamples.onpremiseengine.flowelements.SimpleOnPremiseEngineBuilder; +import java.io.PrintStream; import java.text.SimpleDateFormat; import java.util.Calendar; @@ -72,7 +73,31 @@ public void run() throws Exception { } } + /** + * The rule above and below the "Find out more" section. + */ + private static final String RULE = + "----------------------------------------------------------------"; + + /** + * Ends the output with the pages and repositories that say more about + * what this example shows. + * @param output where to write + */ + static void writeFindOutMore(PrintStream output) { + output.println(); + output.println(RULE); + output.println("Find out more"); + output.println("\tThis example: https://51degrees.com/documentation/_pipeline_api__examples__custom_element__on_premise.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.onpremise-engine-main.java&utm_term=find-out-more"); + output.println("\tCustom flow elements: https://51degrees.com/documentation/_pipeline_api__features__custom_elements.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.onpremise-engine-main.java&utm_term=find-out-more"); + output.println("\thttps://github.com/51Degrees/51degrees-java"); + output.println("\thttps://github.com/51Degrees/specifications"); + output.println(RULE); + output.flush(); + } + public static void main(String[] args) throws Exception { new Example().run(); + writeFindOutMore(System.out); } } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/test/java/pipeline/developerexamples/onpremiseengine/FindOutMoreTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/test/java/pipeline/developerexamples/onpremiseengine/FindOutMoreTests.java new file mode 100644 index 000000000..4acc41328 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/test/java/pipeline/developerexamples/onpremiseengine/FindOutMoreTests.java @@ -0,0 +1,98 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.onpremiseengine; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The example ends its output with a "Find out more" section, which lists + * pages on 51degrees.com, each with its campaign tags, and the repository. + */ +public class FindOutMoreTests { + + private static final String RULE = + "----------------------------------------------------------------"; + + private static final String HEADING = "Find out more"; + + private static final String REPOSITORY = + "https://github.com/51Degrees/51degrees-java"; + + private static final Pattern TAGGED = Pattern.compile( + "https://51degrees\\.com/[^?\\s]*\\?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=[a-z0-9][a-z0-9._-]*&utm_term=find-out-more$"); + + @Test + public void Main_EndsWithFindOutMore() throws Exception { + String output = runMain(); + int example = output.lastIndexOf("your star sign is"); + int heading = output.lastIndexOf(RULE + "\n" + HEADING + "\n"); + assertTrue(example >= 0, "The example wrote nothing:\n" + output); + assertTrue(heading > example, + "No Find out more section after the example's output:\n" + + output); + String section = output.substring( + heading + RULE.length() + HEADING.length() + 2); + int end = section.indexOf(RULE + "\n"); + assertTrue(end > 0, "The section has no closing rule:\n" + output); + section = section.substring(0, end); + int pages = 0; + for (String line : section.split("\n")) { + if (line.contains("51degrees.com")) { + assertTrue(TAGGED.matcher(line).find(), + "A 51degrees.com link without its campaign tags: " + + line); + pages++; + } + } + assertEquals(2, pages, "The pages listed:\n" + section); + assertTrue(section.contains(REPOSITORY), + "The repository is not listed:\n" + section); + } + + /** + * Runs the example as a person would and returns what it wrote, with + * each line ending in a line feed. + */ + private static String runMain() throws Exception { + PrintStream original = System.out; + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + System.setOut(new PrintStream(captured, true, "UTF-8")); + try { + Main.main(new String[0]); + } finally { + System.setOut(original); + } + return new String(captured.toByteArray(), StandardCharsets.UTF_8) + .replace("\r\n", "\n"); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/test/java/pipeline/developerexamples/onpremiseengine/WritesAfterProcessingTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/test/java/pipeline/developerexamples/onpremiseengine/WritesAfterProcessingTests.java new file mode 100644 index 000000000..ce1a5dfc4 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.onpremise-engine/src/test/java/pipeline/developerexamples/onpremiseengine/WritesAfterProcessingTests.java @@ -0,0 +1,111 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.onpremiseengine; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.data.AspectDataBase; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; +import pipeline.developerexamples.onpremiseengine.flowelements.SimpleOnPremiseEngine; +import pipeline.developerexamples.onpremiseengine.flowelements.SimpleOnPremiseEngineBuilder; + +import java.util.Calendar; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * The check, for the Flow Element in this module, the + * {@link SimpleOnPremiseEngine}, that processes it, then writes to its + * Element Data and expects the error. + * @see Specification + */ +public class WritesAfterProcessingTests { + + private static final ILoggerFactory LOGGER_FACTORY = + LoggerFactory.getILoggerFactory(); + + /** + * Every write method of Element Data, and of Aspect Data, fails once its + * Flow Element has finished. + * @param data the Element Data of a finished Flow Element + */ + static void assertWritesFail(ElementData data) { + assertThrows( + IllegalStateException.class, + () -> data.put("after", "value"), + "put"); + assertThrows( + IllegalStateException.class, + () -> data.populateFromMap( + Collections.singletonMap("after", "value")), + "populateFromMap"); + if (data instanceof AspectDataBase) { + AspectDataBase aspect = (AspectDataBase) data; + assertThrows( + IllegalStateException.class, + aspect::setCloudRequestFailed, + "setCloudRequestFailed"); + assertThrows( + IllegalStateException.class, + () -> aspect.addEngine(null), + "addEngine"); + assertThrows( + IllegalStateException.class, + () -> aspect.addProcessCallable(null), + "addProcessCallable"); + } + } + + @Test + public void simpleOnPremiseEngine_ElementDataRefusesWrites() + throws Exception { + SimpleOnPremiseEngine engine = + new SimpleOnPremiseEngineBuilder(LOGGER_FACTORY) + .setAutoUpdate(false) + .build( + WritesAfterProcessingTests.class.getClassLoader() + .getResource("starsigns.csv").getPath(), + false); + Calendar dob = Calendar.getInstance(); + dob.set(1992, Calendar.DECEMBER, 18); + try (Pipeline pipeline = new PipelineBuilder(LOGGER_FACTORY) + .addFlowElement(engine) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("date-of-birth", dob.getTime()); + data.process(); + assertEquals(0, data.getErrors().size(), + String.valueOf(data.getErrors())); + + ElementData own = data.getFromElement(engine); + assertWritesFail(own); + } + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java index f6da01434..cb4ca6e86 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/main/java/pipeline/developerexamples/usagesharing/Main.java @@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.PrintStream; import java.util.Objects; /** @@ -115,7 +116,31 @@ public void run() throws Exception { } } + /** + * The rule above and below the "Find out more" section. + */ + private static final String RULE = + "----------------------------------------------------------------"; + + /** + * Ends the output with the pages and repositories that say more about + * what this example shows. + * @param output where to write + */ + static void writeFindOutMore(PrintStream output) { + output.println(); + output.println(RULE); + output.println("Find out more"); + output.println("\tThis example: https://51degrees.com/documentation/_pipeline_api__examples__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.usage-sharing-main.java&utm_term=find-out-more"); + output.println("\tUsage sharing: https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=pipeline.developer-examples.usage-sharing-main.java&utm_term=find-out-more"); + output.println("\thttps://github.com/51Degrees/51degrees-java"); + output.println("\thttps://github.com/51Degrees/specifications"); + output.println(RULE); + output.flush(); + } + public static void main(String[] args) throws Exception { new Example().run(); + writeFindOutMore(System.out); } } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/test/java/pipeline/developerexamples/usagesharing/FindOutMoreTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/test/java/pipeline/developerexamples/usagesharing/FindOutMoreTests.java new file mode 100644 index 000000000..9155c5097 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.usage-sharing/src/test/java/pipeline/developerexamples/usagesharing/FindOutMoreTests.java @@ -0,0 +1,100 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.usagesharing; + +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * The example ends its output with a "Find out more" section, which lists + * pages on 51degrees.com, each with its campaign tags, and the repository. + */ +public class FindOutMoreTests { + + private static final String RULE = + "----------------------------------------------------------------"; + + private static final String HEADING = "Find out more"; + + private static final String REPOSITORY = + "https://github.com/51Degrees/51degrees-java"; + + private static final Pattern TAGGED = Pattern.compile( + "https://51degrees\\.com/[^?\\s]*\\?utm_source=code" + + "&utm_medium=example&utm_campaign=51degrees-java" + + "&utm_content=[a-z0-9][a-z0-9._-]*&utm_term=find-out-more$"); + + @Test + public void Main_EndsWithFindOutMore() throws Exception { + String output = runMain(); + int example = output.lastIndexOf("Pipeline created with share usage element"); + int heading = output.lastIndexOf(RULE + "\n" + HEADING + "\n"); + assertTrue("The example wrote nothing:\n" + output, example >= 0); + assertTrue( + "No Find out more section after the example's output:\n" + + output, + heading > example); + String section = output.substring( + heading + RULE.length() + HEADING.length() + 2); + int end = section.indexOf(RULE + "\n"); + assertTrue("The section has no closing rule:\n" + output, end > 0); + section = section.substring(0, end); + int pages = 0; + for (String line : section.split("\n")) { + if (line.contains("51degrees.com")) { + assertTrue( + "A 51degrees.com link without its campaign tags: " + + line, + TAGGED.matcher(line).find()); + pages++; + } + } + assertEquals("The pages listed:\n" + section, 2, pages); + assertTrue("The repository is not listed:\n" + section, + section.contains(REPOSITORY)); + } + + /** + * Runs the example as a person would and returns what it wrote, with + * each line ending in a line feed. + */ + private static String runMain() throws Exception { + PrintStream original = System.out; + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + System.setOut(new PrintStream(captured, true, "UTF-8")); + try { + Main.main(new String[0]); + } finally { + System.setOut(original); + } + return new String(captured.toByteArray(), StandardCharsets.UTF_8) + .replace("\r\n", "\n"); + } +} diff --git a/pipeline/pipeline.developer-examples/pom.xml b/pipeline/pipeline.developer-examples/pom.xml index 08197b56e..b30007c94 100644 --- a/pipeline/pipeline.developer-examples/pom.xml +++ b/pipeline/pipeline.developer-examples/pom.xml @@ -44,6 +44,7 @@ pipeline.developer-examples.clientside-element-mvc pipeline.developer-examples.usage-sharing pipeline.developer-examples.fodid + pipeline.developer-examples.derivedproperty true diff --git a/pipeline/pipeline.did/pom.xml b/pipeline/pipeline.did/pom.xml index c5ba43ddc..a925f2780 100644 --- a/pipeline/pipeline.did/pom.xml +++ b/pipeline/pipeline.did/pom.xml @@ -153,4 +153,30 @@ + + + + owid-dangling-doc-comments + + [23,) + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + -Xlint:-dangling-doc-comments + + + + + + + + diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java index 9c2d067d2..88c1bff14 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java @@ -59,9 +59,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Logger; /** * The client for everything a server does with a 51Did against the @@ -120,8 +118,8 @@ public final class DidClient { "_51DEGREES_CLOUD_ENDPOINT"; /** - * The old name of the environment variable read for the API base, read - * only when {@link #CLOUD_ENDPOINT_VARIABLE} is not set. + * The old name of the environment variable read for the API base. It + * is no longer read. * * @deprecated use {@link #CLOUD_ENDPOINT_VARIABLE}, and set the * environment variable it names in place of this one. @@ -149,13 +147,6 @@ public final class DidClient { private static final String USER_AGENT = "pipeline.did/" + version(); - /** - * Set once the use of {@link #ENDPOINT_VARIABLE} has been logged, so - * that a process creating many clients logs it once. - */ - private static final AtomicBoolean OLD_VARIABLE_LOGGED = - new AtomicBoolean(false); - /** * Longest encoded identifier this client will take from a caller. The * figure is arbitrary and deliberately generous, far above anything the @@ -1173,29 +1164,14 @@ static String encode(String value) { /** * The API base to use: the argument, else - * {@value #CLOUD_ENDPOINT_VARIABLE}, else the deprecated - * {@value #ENDPOINT_VARIABLE}, else {@link #DEFAULT_ENDPOINT}, ending - * in exactly one slash so every route can be appended directly. The - * first use of the deprecated variable in the process is logged as a - * warning. + * {@value #CLOUD_ENDPOINT_VARIABLE}, else {@link #DEFAULT_ENDPOINT}, + * ending in exactly one slash so every route can be appended directly. */ static String resolveEndpoint(String endpoint) { String value = blankToNull(endpoint); if (value == null) { value = blankToNull(System.getenv(CLOUD_ENDPOINT_VARIABLE)); } - if (value == null) { - value = blankToNull(System.getenv(ENDPOINT_VARIABLE)); - if (value != null - && OLD_VARIABLE_LOGGED.compareAndSet(false, true)) { - Logger.getLogger(DidClient.class.getName()).warning( - "The environment variable " + ENDPOINT_VARIABLE - + " is deprecated and " + CLOUD_ENDPOINT_VARIABLE - + " replaces it. Set " + CLOUD_ENDPOINT_VARIABLE - + " to the same value, as a later release may no " - + "longer read " + ENDPOINT_VARIABLE + "."); - } - } if (value == null) { value = DEFAULT_ENDPOINT; } diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodId.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodId.java index e94ef24d2..7af5ed21d 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodId.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodId.java @@ -257,29 +257,35 @@ private static FodIdParseResult read(OwidParseResult envelope) { * already read or signed. This is the one walk of the payload that every * reader, throwing or not, goes through. *

    - * The rules are lower bounds only. The header must be present before the - * type can be read, and the type then sets the least the payload can - * hold. Anything longer is accepted as it stands, because the bytes past - * the match key are the Terms and then a creator context section whose - * shape the cloud judges. The Terms adds nothing to those bounds, since - * a payload that ends at the match key reads as a Terms of zero. + * The version is read from the first byte before anything else, so a + * payload of a version this package does not know is refused naming + * that version however short it is. The other rules are lower bounds + * only. The header must be present before the type can be read, and the + * type then sets the least the payload can hold. Anything longer is + * accepted as it stands, because the bytes past the match key are the + * Terms and then a creator context section whose shape the cloud + * judges. The Terms adds nothing to those bounds, since a payload that + * ends at the match key reads as a Terms of zero. */ private static FodIdParseResult read(Owid owid) { byte[] payload = owid.getPayload(); - if (payload.length < HEADER_LENGTH) { + if (payload.length <= FLAGS_OFFSET) { return FodIdParseResult.failed(FodIdParseStatus.PAYLOAD_TOO_SHORT); } int flags = payload[FLAGS_OFFSET] & 0xFF; - // The version is read before any field, because a later version - // exists precisely because a field moved. Reading a payload of a - // version this package does not know under the layout it does know - // would answer with values that are wrong rather than absent, - // which is worse than refusing, and a version that nothing checks - // protects nothing. + // The version is read before any field, and before the header's + // length is checked, because a later version exists precisely + // because a field moved. Reading a payload of a version this package + // does not know under the layout it does know would answer with + // values that are wrong rather than absent, which is worse than + // refusing, and a version that nothing checks protects nothing. int payloadVersion = payloadVersionOf(flags); if (payloadVersion != SUPPORTED_PAYLOAD_VERSION) { return FodIdParseResult.unsupportedPayloadVersion(payloadVersion); } + if (payload.length < HEADER_LENGTH) { + return FodIdParseResult.failed(FodIdParseStatus.PAYLOAD_TOO_SHORT); + } // A payload with no usage bit set states no usage at all. The cloud // never writes one, so it is damaged or forged, and it is refused // here rather than read, because the only safe answer to it is not diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/RedeemResult.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/RedeemResult.java index a2625f942..fbeeca1b0 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/RedeemResult.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/RedeemResult.java @@ -28,6 +28,7 @@ import java.time.format.DateTimeParseException; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.Locale; import java.util.Map; /** @@ -196,15 +197,17 @@ public String getValue() { * Verified, misconfigured and notrecorded are each read on their own, * because none of them is a mismatch. Anything else, including a word * this client does not know, is a mismatch, so an unexpected value - * never reads as a pass. + * never reads as a pass. The comparison ignores case, so the word is + * lower-cased in the root locale first. * * @param value the factor's string from the cloud, or null * @return the matching outcome, or {@link #MISMATCH} */ public static Factor fromValue(String value) { if (value != null) { + String word = value.toLowerCase(Locale.ROOT); for (Factor factor : values()) { - if (factor.value.equals(value)) { + if (factor.value.equals(word)) { return factor; } } diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientEnvironmentTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientEnvironmentTests.java index 562bbddb2..ecf6f6f56 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientEnvironmentTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientEnvironmentTests.java @@ -50,50 +50,31 @@ public class DidClientEnvironmentTests { private static final String OLD_NAME = "FOD_CLOUD_API_URL"; private static final String NEW_ENDPOINT = "https://new.example/api/v4/"; - private static final String OLD_ENDPOINT = "https://old.example/api/v4/"; private static final int PROBE_TIMEOUT_SECONDS = 120; @Test - public void oldNameAlone_GivesItsEndpoint() throws Exception { - List output = probe(null, OLD_ENDPOINT); + public void newName_GivesItsEndpoint() throws Exception { + List output = probe(NEW_ENDPOINT); assertEquals( - Arrays.asList(OLD_ENDPOINT, OLD_ENDPOINT), endpoints(output)); - } - - @Test - public void oldNameAlone_IsLoggedAsDeprecatedOnce() throws Exception { - List logged = logged(probe(null, OLD_ENDPOINT)); - - assertEquals(logged.toString(), 1, logged.size()); - String record = logged.get(0); - assertTrue(record, record.startsWith("WARNING ")); - assertTrue(record, record.contains(OLD_NAME)); - assertTrue(record, record.contains(NEW_NAME)); - assertTrue(record, record.contains("deprecated")); - } - - @Test - public void newNameBlank_OldNameIsRead() throws Exception { - List output = probe(" ", OLD_ENDPOINT); - - assertEquals( - Arrays.asList(OLD_ENDPOINT, OLD_ENDPOINT), endpoints(output)); + Arrays.asList(NEW_ENDPOINT, NEW_ENDPOINT), endpoints(output)); + assertEquals(output.toString(), 0, logged(output).size()); } @Test - public void bothNames_NewNameWins() throws Exception { - List output = probe(NEW_ENDPOINT, OLD_ENDPOINT); + public void newNameBlank_GivesTheDefault() throws Exception { + List output = probe(" "); assertEquals( - Arrays.asList(NEW_ENDPOINT, NEW_ENDPOINT), endpoints(output)); - assertEquals(output.toString(), 0, logged(output).size()); + Arrays.asList( + DidClient.DEFAULT_ENDPOINT, DidClient.DEFAULT_ENDPOINT), + endpoints(output)); } @Test public void neitherName_GivesTheDefault() throws Exception { - List output = probe(null, null); + List output = probe(null); assertEquals( Arrays.asList( @@ -114,10 +95,10 @@ public void oldConstant_KeepsItsNameAndValue() throws Exception { } /** - * Runs the probe with each variable set to the value given, or absent + * Runs the probe with the variable set to the value given, or absent * where the value is null, and returns the lines it printed. */ - private static List probe(String newValue, String oldValue) + private static List probe(String newValue) throws Exception { String java = Paths.get( System.getProperty("java.home"), "bin", "java").toString(); @@ -126,7 +107,6 @@ private static List probe(String newValue, String oldValue) Map environment = builder.environment(); environment.put("CLASSPATH", classPath()); set(environment, NEW_NAME, newValue); - set(environment, OLD_NAME, oldValue); builder.redirectErrorStream(true); Process process = builder.start(); diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java index 6794f4ee8..b6adfa766 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java @@ -46,11 +46,10 @@ /** * Tests against the live cloud service. They need a resource key, read - * from the names {@link LiveCloudKeys} lists, and they fail rather than - * skip when there is none, because a run that called nothing must not - * report success. {@code _51DEGREES_CLOUD_ENDPOINT} points them at another - * host, and a licence key is taken from the names {@link LiveCloudKeys} lists - * where the account holds one. Each test creates a 51Did through the + * from the name {@link LiveCloudKeys} gives, and they are skipped, naming + * that variable, when there is none. {@code _51DEGREES_CLOUD_ENDPOINT} + * points them at another host, and the 51Did license key is taken from the + * name {@link LiveCloudKeys} gives where the account holds one. Each test creates a 51Did through the * cloud {@code json} endpoint, which is one use against the resource key, * plus one for each cloud call it then makes. *

    @@ -67,9 +66,7 @@ public class DidClientLiveTests { @Before public void init() { resourceKey = LiveCloudKeys.find(LiveCloudKeys.RESOURCE_KEY_NAMES); - if (resourceKey == null) { - fail(LiveCloudKeys.noResourceKey()); - } + Assume.assumeTrue(LiveCloudKeys.noResourceKey(), resourceKey != null); // One lookup serves both, so the key the form body sends and the // one hasLicenceKey() answers for can never disagree. licenceKey = LiveCloudKeys.find(LiveCloudKeys.LICENCE_KEY_NAMES); @@ -133,7 +130,7 @@ public void redeem_GarbageResult_IsUnreadable() throws Exception { public void verifyFullThenRedeemWithTheLicence_ReadsTheVerdict() throws Exception { Assume.assumeTrue( - "Set _51DEGREES_LICENSE_KEY to redeem a sealed result.", + "Set _51DEGREES_LICENSE_KEY_51DID to redeem a sealed result.", client.hasLicenceKey()); List identifiers = identifiersFor("id.usage", "standard"); @@ -206,7 +203,7 @@ public void verifyFullThenRedeemWithTheLicence_ReadsTheVerdict() public void anIdentifierForAnotherAddress_ReportsEveryFactor() throws Exception { Assume.assumeTrue( - "Set _51DEGREES_LICENSE_KEY to redeem a sealed result.", + "Set _51DEGREES_LICENSE_KEY_51DID to redeem a sealed result.", client.hasLicenceKey()); List identifiers = identifiersCreatedFor( diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java index 501a2cabf..6f8409f6e 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java @@ -42,6 +42,7 @@ import java.util.Collections; import java.util.Deque; import java.util.List; +import java.util.Locale; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.Executor; @@ -129,13 +130,9 @@ public void endpoint_IsNormalisedToOneTrailingSlash() { } @Test - @SuppressWarnings("deprecation") public void endpoint_DefaultsToTheCloudOrTheEnvironment() { String fromEnvironment = System.getenv(DidClient.CLOUD_ENDPOINT_VARIABLE); - if (fromEnvironment == null || fromEnvironment.trim().isEmpty()) { - fromEnvironment = System.getenv(DidClient.ENDPOINT_VARIABLE); - } String expected = fromEnvironment == null || fromEnvironment.trim().isEmpty() ? DidClient.DEFAULT_ENDPOINT @@ -1038,6 +1035,45 @@ public void redeem_UnknownFactorValue_IsStillAMismatch() result.getFactors().get("transport")); } + /** + * The comparison of a factor word ignores case, so each word in other + * letters reads as its own outcome rather than as a mismatch. The + * default locale is Turkish, where lower-casing an I in the default + * locale gives a dotless i. + */ + @Test + public void redeem_FactorWordsInOtherLetters_ReadAsTheirOutcome() + throws Exception { + Locale saved = Locale.getDefault(); + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + try { + transport.queue(200, "{\"signature\":\"verified\"," + + "\"context\":\"mismatch\"," + + "\"factors\":{\"transport\":\"Verified\"," + + "\"device\":\"NotRecorded\"," + + "\"browserip\":\"MISCONFIGURED\"," + + "\"connectionip\":\"VERIFIED\"," + + "\"asn\":\"Mismatch\"}}"); + FodId fodId = key2.fodIdAt(canonicalPayload(), WEEK2); + + RedeemResult result = + client.redeem(fodId, "sealed", "abc").join(); + + assertEquals(RedeemResult.Factor.VERIFIED, + result.getFactors().get("transport")); + assertEquals(RedeemResult.Factor.NOT_RECORDED, + result.getFactors().get("device")); + assertEquals(RedeemResult.Factor.MISCONFIGURED, + result.getFactors().get("browserip")); + assertEquals(RedeemResult.Factor.VERIFIED, + result.getFactors().get("connectionip")); + assertEquals(RedeemResult.Factor.MISMATCH, + result.getFactors().get("asn")); + } finally { + Locale.setDefault(saved); + } + } + /** Every outcome reports the cloud's own word for itself. */ @Test public void factor_GetValue_IsTheCloudsWord() { diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdParseTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdParseTests.java index fec3bafaa..360f9ca70 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdParseTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdParseTests.java @@ -491,6 +491,47 @@ public void version_NotZero_IsRefused() throws Exception { } } + /** + * The version is read from the first byte, before the layout that sets + * the header's length, so a payload too short for that header is still + * refused naming the version it gives. A short payload of version 0 is + * too short. + */ + @Test + public void version_ShortPayload_IsRefusedNamingTheVersion() + throws Exception { + for (int length = 1; length < FodId.HEADER_LENGTH; length++) { + byte[] shortPayload = Arrays.copyOf(canonicalPayload(), length); + + assertEquals("length " + length + " version 0", + FodIdParseStatus.PAYLOAD_TOO_SHORT, + FodId.tryFromBase64(factory + .signedOwidAt(withVersion(shortPayload, 0), DATE) + .asBase64()).getStatus()); + + for (int version : new int[] { 1, 2, 3 }) { + String base64 = factory + .signedOwidAt(withVersion(shortPayload, version), DATE) + .asBase64(); + String label = "length " + length + " version " + version; + + FodIdParseResult result = FodId.tryFromBase64(base64); + assertEquals(label, + FodIdParseStatus.UNSUPPORTED_PAYLOAD_VERSION, + result.getStatus()); + assertEquals(label, version, result.getPayloadVersion()); + assertNull(label, result.getValue()); + try { + FodId.fromBase64(base64); + fail(label + " should have been refused"); + } catch (IllegalArgumentException thrown) { + assertTrue(label + ": " + thrown.getMessage(), + thrown.getMessage().contains("version " + version)); + } + } + } + } + /** * The throwing readers name the version they found, so whoever reads * the message knows which layout the identifier claims rather than only diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdSurfaceTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdSurfaceTests.java new file mode 100644 index 000000000..d8865e67a --- /dev/null +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdSurfaceTests.java @@ -0,0 +1,131 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.did; + +import com.swancommunity.owid.Owid; +import com.swancommunity.owid.Version; +import org.junit.Test; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Member; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Checks that {@link FodId} keeps the byte layout out of its public + * surface, has no member for the payload version, and is only ever made by + * reading a complete Envelope, as the 51Did specification asks. + */ +public class FodIdSurfaceTests { + + /** + * True when a caller outside the package can reach the member. + */ + private static boolean isVisible(Member member) { + int modifiers = member.getModifiers(); + return Modifier.isPublic(modifiers) || Modifier.isProtected(modifiers); + } + + /** + * did-specification/README.md lines 33 to 35, "Application code MUST + * NOT read the layout by hand, and the packages therefore keep the + * offsets and lengths out of their public surface." + */ + @Test + public void layout_OffsetsAndLengths_AreNotPublic() { + List visible = new ArrayList<>(); + for (Field field : FodId.class.getDeclaredFields()) { + if (isVisible(field)) { + visible.add(field.getName()); + } + } + assertEquals( + "FodId fields a caller can reach", + new ArrayList(), + visible); + } + + /** + * did-specification/package-surface.md lines 55 and 56, "The version + * belongs to the refusal and MUST NOT appear as a member of the + * identifier". The only version a caller reads from a FodId is the + * Envelope's own, through getVersion. + */ + @Test + public void payloadVersion_IsNotAMemberOfTheIdentifier() { + List visible = new ArrayList<>(); + for (Method method : FodId.class.getDeclaredMethods()) { + boolean namesVersion = method.getName() + .toLowerCase(Locale.ROOT) + .contains("version"); + if (isVisible(method) && + namesVersion && + method.getReturnType() != Version.class) { + visible.add(method.getName()); + } + } + assertEquals( + "FodId methods that give a version other than the Envelope's", + new ArrayList(), + visible); + } + + /** + * did-specification/identifier-layout.md lines 42 and 43, "a 51Did + * package MUST create instances only by reading bytes that are already + * a complete Envelope." + */ + @Test + public void instances_AreMadeOnlyByReadingAnEnvelope() { + assertTrue( + "FodId is final, so no subclass can make an instance", + Modifier.isFinal(FodId.class.getModifiers())); + for (Constructor constructor : + FodId.class.getDeclaredConstructors()) { + assertTrue( + "constructor " + constructor + " is private", + Modifier.isPrivate(constructor.getModifiers())); + } + List> readable = Arrays.>asList( + String.class, byte[].class, Owid.class); + for (Method method : FodId.class.getDeclaredMethods()) { + if (isVisible(method) && + Modifier.isStatic(method.getModifiers()) && + method.getReturnType() == FodId.class) { + Class[] parameters = method.getParameterTypes(); + assertTrue( + method.getName() + " takes one encoded Envelope", + parameters.length == 1 && + readable.contains(parameters[0])); + } + } + } +} diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java index 88b51e27e..6185b307e 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java @@ -22,9 +22,6 @@ package fiftyone.pipeline.did; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; /** * Where the tests that call the live 51Degrees cloud service read their @@ -33,77 +30,41 @@ *

    * Every name is tried as an environment variable first and then as a * system property, so a developer can export one and a build can pass one - * on the command line with {@code -D}. There is no fallback to a skip, - * because a cloud test that quietly passes with no key reports success - * having called nothing. + * on the command line with {@code -D}. Each key is read under its one + * _51DEGREES_ name and no other. A test with no key is skipped, naming + * the variable to set, rather than passed having called nothing. */ final class LiveCloudKeys { /** - * The resource key names, in the order they are tried. - *

    - * {@code _51DEGREES_RESOURCE_KEY} is first because the workflow that - * runs these tests once for every key the organisation holds sets - * that single name to each key in turn, and a name later in this list - * winning would make every one of those runs use the same key. + * The name the resource key is read from. The workflow that runs these + * tests once for every key the organisation holds sets this single + * name to each key in turn. */ static final String[] RESOURCE_KEY_NAMES = { "_51DEGREES_RESOURCE_KEY", - "_51DEGREES_RESOURCE_KEY_BESPOKE", - "TestResourceKey", - "SUPER_RESOURCE_KEY", }; /** - * The licence key names, in the order they are tried. A licence key - * is optional, because an account can hold its products on the - * resource key alone. + * The name the license key is read from. A license key is optional, + * because an account can hold its products on the resource key alone. + * It is the 51Did license, which redeems a sealed verdict, and not + * _51DEGREES_LICENSE_KEY, which holds the license for the data files. */ static final String[] LICENCE_KEY_NAMES = { - "_51DEGREES_LICENSE_KEY", - "_51DEGREES_LICENSE_KEY_BESPOKE", - "_51DEGREES_LICENSE_KEY_CLOUDV5BESPOKE", - "TestLicenseKey", + "_51DEGREES_LICENSE_KEY_51DID", }; private LiveCloudKeys() { } - /** - * The names that have a deprecated old name, each followed by it. The - * old name is read only when its new name is not set. - */ - private static final String[][] DEPRECATED_NAMES = { - {"_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY"}, - {"_51DEGREES_LICENSE_KEY", "LICENSE_KEY"}, - }; - - /** The old names whose use has been reported, each reported once. */ - private static final Set REPORTED_OLD_NAMES = - Collections.synchronizedSet(new HashSet()); - /** * The first value set under any of the given names, or null when none - * of them is set anywhere. A name with a deprecated old name falls - * back to it, saying so on standard error the first time. + * of them is set anywhere. */ static String find(String[] names) { for (String name : names) { String value = read(name); - if (value == null) { - String oldName = oldNameOf(name); - if (oldName != null) { - value = read(oldName); - if (value != null && REPORTED_OLD_NAMES.add(oldName)) { - System.err.println( - "The environment variable " + oldName - + " is deprecated and " + name - + " replaces it. Set " + name - + " to the same value, as a later release may no longer " - + "read " + oldName + "."); - } - } - } if (value != null) { return value; } @@ -124,28 +85,19 @@ private static String read(String name) { return value == null || value.trim().isEmpty() ? null : value.trim(); } - private static String oldNameOf(String name) { - for (String[] pair : DEPRECATED_NAMES) { - if (pair[0].equals(name)) { - return pair[1]; - } - } - return null; - } - /** - * What to fail with when no resource key is set, naming every place - * that was looked and what the key has to be entitled to. + * Why the tests are skipped when no resource key is set, naming the + * variable to set and what the key has to be entitled to. */ static String noResourceKey() { return "These tests call the live 51Degrees cloud service, so they " - + "need a resource key and there is nothing to fall back on. " - + "None of " + String.join(", ", RESOURCE_KEY_NAMES) + " is set, " - + "as an environment variable or as a system property, tried in " - + "that order. The key has to carry the fodid product, because " + + "need a resource key, and they are skipped because " + + String.join(", ", RESOURCE_KEY_NAMES) + " is not set as an " + + "environment variable or as a system property. " + + "The key has to carry the fodid product, because " + "these tests ask the cloud for FODiD.IdProbGlobal and " + "FODiD.IdProbLic. Create a key at " - + "https://configure.51degrees.com."; + + "https://51degrees.com/configure."; } /** @@ -158,6 +110,6 @@ static String notEntitled(String detail) { + "51Did for " + detail + ". That is the resource key not " + "carrying the fodid product rather than a fault in the " + "service or in this package. Use a key entitled to fodid, " - + "from https://configure.51degrees.com."; + + "from https://51degrees.com/configure."; } } diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeysTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeysTests.java index bf0f5a370..ee605b231 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeysTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeysTests.java @@ -22,84 +22,38 @@ package fiftyone.pipeline.did; -import org.junit.After; -import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assume.assumeTrue; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertTrue; /** - * Tests which names {@link LiveCloudKeys} reads a key from. System - * properties stand in for the environment variables, because the lookup - * reads either and a test cannot set an environment variable. + * Each key the live 51Did tests use is read under its one _51DEGREES_ + * name, and no other name is read in its place. */ public class LiveCloudKeysTests { - private static final String RESOURCE_KEY_NAME = "_51DEGREES_RESOURCE_KEY"; - private static final String OLD_RESOURCE_KEY_NAME = "RESOURCE_KEY"; - private static final String LICENCE_KEY_NAME = "_51DEGREES_LICENSE_KEY"; - private static final String OLD_LICENCE_KEY_NAME = "LICENSE_KEY"; - private static final String OTHER_NAME = "TestResourceKey"; - - private static final String[] NAMES = { - RESOURCE_KEY_NAME, OLD_RESOURCE_KEY_NAME, - LICENCE_KEY_NAME, OLD_LICENCE_KEY_NAME, OTHER_NAME, - }; - - private final String[] saved = new String[NAMES.length]; - - @Before - public void clearNames() { - for (int i = 0; i < NAMES.length; i++) { - // An environment variable would answer before the system - // properties set here, so the test cannot run. - assumeTrue(System.getenv(NAMES[i]) == null); - saved[i] = System.clearProperty(NAMES[i]); - } - } - - @After - public void restoreNames() { - for (int i = 0; i < NAMES.length; i++) { - if (saved[i] == null) { - System.clearProperty(NAMES[i]); - } else { - System.setProperty(NAMES[i], saved[i]); - } - } - } - @Test - public void resourceKey_OldNameWhenNewIsNotSet() { - System.setProperty(OLD_RESOURCE_KEY_NAME, "old-name-key"); - - assertEquals("old-name-key", - LiveCloudKeys.find(new String[] {RESOURCE_KEY_NAME})); + public void resourceKey_ReadUnderItsOwnNameOnly() { + assertArrayEquals(new String[]{"_51DEGREES_RESOURCE_KEY"}, + LiveCloudKeys.RESOURCE_KEY_NAMES); } + /** + * A sealed verdict is redeemed with the 51Did license. + * _51DEGREES_LICENSE_KEY holds the data file license, so it is not + * read here. + */ @Test - public void licenceKey_OldNameWhenNewIsNotSet() { - System.setProperty(OLD_LICENCE_KEY_NAME, "old-name-licence"); - - assertEquals("old-name-licence", - LiveCloudKeys.find(new String[] {LICENCE_KEY_NAME})); + public void licenceKey_IsThe51DidLicence() { + assertArrayEquals(new String[]{"_51DEGREES_LICENSE_KEY_51DID"}, + LiveCloudKeys.LICENCE_KEY_NAMES); } @Test - public void newNameWinsOverOld() { - System.setProperty(RESOURCE_KEY_NAME, "new-name-key"); - System.setProperty(OLD_RESOURCE_KEY_NAME, "old-name-key"); - - assertEquals("new-name-key", - LiveCloudKeys.find(new String[] {RESOURCE_KEY_NAME})); - } - - @Test - public void oldNameIsReadOnlyForItsNewName() { - System.setProperty(OLD_RESOURCE_KEY_NAME, "old-name-key"); - - assertNull(LiveCloudKeys.find(new String[] {OTHER_NAME})); + public void noResourceKey_NamesTheVariable() { + assertTrue(LiveCloudKeys.noResourceKey(), + LiveCloudKeys.noResourceKey().contains( + "_51DEGREES_RESOURCE_KEY is not set")); } } diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java index 3bfd42e8f..da4c74198 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java @@ -29,9 +29,11 @@ import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Set; import static fiftyone.pipeline.core.Constants.*; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_DEFAULT_SHARE_ALL_QUERY_STRING_PARAMETERS; /** * This filter is used by the {@link ShareUsageElement} to decide which of the @@ -43,12 +45,16 @@ * 2) A cookie that starts with {@link Constants#FIFTYONE_COOKIE_PREFIX} or is * the session cookie (if configured in the constructor). * 3) A session value that starts with {@link Constants#FIFTYONE_COOKIE_PREFIX}. - * 4) A query string parameter that starts with - * {@link Constants#FIFTYONE_COOKIE_PREFIX} or has been configured to be - * shared using the constructor parameter. + * 4) A query string or HTTP form parameter that starts with + * {@link Constants#FIFTYONE_COOKIE_PREFIX}, or that is Evidence the + * Pipeline uses, being a key a Flow Element in the Pipeline accepts, + * whatever the configuration. The included query string parameters and + * share all query string parameters options are deprecated and have no + * effect. * 5) Not a header, cookie, session value or query string parameter. - * When all Evidence is shared, everything is included. When all query string - * parameters are shared, every query string parameter is included. + * When all Evidence is shared, everything is included but cookies and query + * string and form parameters, which follow rules 2 and 4 whatever the + * configuration. * The prefix of a key is everything before its first '.', and the field * everything after it. An email address, the salt for hashed email * identifiers, the query string, the path and the Cookie header are never @@ -74,10 +80,6 @@ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { */ private final boolean shareAllEvidence; - /** - * If true then every query string parameter is included. - */ - private final boolean shareAllQueryStringParameters; /** * The content of HTTP headers in this array will not be included in the @@ -87,11 +89,6 @@ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { */ private final Set blockedHttpHeaders = new HashSet<>(); - /** - * Query string parameters will not be shared by default. Any query string - * parameters to be shared must be added to this collection. - */ - private final Set includedQueryStringParams = new HashSet<>(); /** * Evidence keys that are never shared, matched without regard to case as @@ -111,12 +108,10 @@ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { }; /** - * Constructor. + * Constructor. All Evidence is not shared, as is the default. * @param blockedHttpHeaders a list of the names of the HTTP headers that * share usage should not send to 51Degrees - * @param includedQueryStringParams a list of the names of query string - * parameters that share usage should send - * to 51Degrees + * @param includedQueryStringParams deprecated, and has no effect * @param includeSession if true then the session cookie will be included in * the filter * @param sessionCookieName the name of the cookie that contains the session @@ -133,24 +128,22 @@ public EvidenceKeyFilterShareUsage( includeSession, sessionCookieName, false, - false); + SHARE_USAGE_DEFAULT_SHARE_ALL_QUERY_STRING_PARAMETERS); } /** * Constructor. * @param blockedHttpHeaders a list of the names of the HTTP headers that * share usage should not send to 51Degrees - * @param includedQueryStringParams a list of the names of query string - * parameters that share usage should send - * to 51Degrees + * @param includedQueryStringParams deprecated, and has no effect * @param includeSession if true then the session cookie will be included in * the filter * @param sessionCookieName the name of the cookie that contains the session * id * @param shareAllEvidence if true then all Evidence is included, apart from - * the keys that are never shared - * @param shareAllQueryStringParameters if true then all query string - * parameters are included + * the keys that are never shared and the cookies + * other than the 51Degrees ones + * @param shareAllQueryStringParameters deprecated, and has no effect */ public EvidenceKeyFilterShareUsage( List blockedHttpHeaders, @@ -160,36 +153,41 @@ public EvidenceKeyFilterShareUsage( boolean shareAllEvidence, boolean shareAllQueryStringParameters) { this.shareAllEvidence = shareAllEvidence; - this.shareAllQueryStringParameters = shareAllQueryStringParameters; // The Cookie header carries every cookie, including the ones the // cookie rule leaves out. - this.blockedHttpHeaders.add("cookies"); this.blockedHttpHeaders.add("cookie"); this.includeSession = includeSession; this.sessionCookieName = sessionCookieName == null ? - null : sessionCookieName.toLowerCase(); + null : sessionCookieName.toLowerCase(Locale.ROOT); for (String header : blockedHttpHeaders) { - String lowerHeader = header.toLowerCase(); + String lowerHeader = header.toLowerCase(Locale.ROOT); if (!this.blockedHttpHeaders.contains(lowerHeader)) { this.blockedHttpHeaders.add(lowerHeader); } } - for (String parameter : includedQueryStringParams) { - String lowerParameter = parameter.toLowerCase(); - if (!this.includedQueryStringParams.contains(lowerParameter)) { - this.includedQueryStringParams.add(lowerParameter); - } - } } @Override public boolean include(String key) { + return include(key, null); + } + + /** + * Whether the value with this key is shared. + * @param key the Evidence key + * @param pipelineUses the Evidence key filter of the Pipeline the value + * came through, which decides whether a query string + * or form parameter is Evidence the Pipeline uses, or + * null if there is none + * @return true if the value is shared + */ + public boolean include(String key, EvidenceKeyFilter pipelineUses) { // The Usage Sharing Element reads the whole web request itself, and // the request is never a value to share. if (ShareUsageEvidence.HTTP_REQUEST_KEY.equalsIgnoreCase(key)) { return false; } - String lowerKey = key.toLowerCase(); + String lowerKey = key.toLowerCase(Locale.ROOT); for (String neverShared : NEVER_SHARED_EVIDENCE_KEYS) { if (lowerKey.equals(neverShared) || lowerKey.endsWith(EVIDENCE_SEPERATOR + neverShared)) { @@ -197,12 +195,32 @@ public boolean include(String key) { } } + int separator = lowerKey.indexOf(EVIDENCE_SEPERATOR); + + // Only the 51Degrees cookies are shared, even when all Evidence is. + // The session cookie is included only for the tracker to compare. + if (separator > 0 && + EVIDENCE_COOKIE_PREFIX.equals(lowerKey.substring(0, separator))) { + String field = lowerKey.substring(separator + 1); + return field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || + (includeSession && field.equals(sessionCookieName)); + } + + // A query string or form parameter is shared only when its name + // starts with 51D_ or the Pipeline uses it, even when all Evidence + // is, so that a parameter such as a password is never shared. + if (separator > 0 && + EVIDENCE_QUERY_PREFIX.equals(lowerKey.substring(0, separator))) { + String field = lowerKey.substring(separator + 1); + return field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || + (pipelineUses != null && pipelineUses.include(key)); + } + if (shareAllEvidence) { return true; } boolean result; - int separator = lowerKey.indexOf(EVIDENCE_SEPERATOR); if (separator > 0) { String prefix = lowerKey.substring(0, separator); String field = lowerKey.substring(separator + 1); @@ -212,25 +230,11 @@ public boolean include(String key) { // appear in the list of blocked headers. result = blockedHttpHeaders.contains(field) == false; break; - case EVIDENCE_COOKIE_PREFIX: - // Only add cookies that start with the 51Degrees cookie - // prefix. - result = field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || - (includeSession && field.equals(sessionCookieName)); - break; case EVIDENCE_SESSION_PREFIX: // Only add session values that start with the 51Degrees cookie // prefix. result = field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX); break; - case EVIDENCE_QUERY_PREFIX: - // Only include query string parameters that start with the - // 51Degrees prefix or have been specified in the - // constructor, unless all of them are shared. - result = shareAllQueryStringParameters || - field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || - includedQueryStringParams.contains(field); - break; default: // Add anything that is not a cookie or a header. result = true; diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java index 61c2c348c..65f12dba8 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.engines.fiftyone.data; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; import fiftyone.pipeline.engines.fiftyone.trackers.ShareUsageTracker; import java.util.List; @@ -69,7 +70,8 @@ public EvidenceKeyFilterShareUsageTracker( * @param sessionCookieName the name of the cookie that contains the session * id * @param shareAllEvidence if true then all Evidence is included, apart from - * the keys that are never shared + * the keys that are never shared and the cookies + * other than the 51Degrees ones * @param shareAllQueryStringParameters if true then all query string * parameters are included */ @@ -91,11 +93,20 @@ public EvidenceKeyFilterShareUsageTracker( @Override public boolean include(String key) { + return include(key, null); + } + + /** + * As the shared values, but never the session id or sequence, which + * change on every request of a session. + */ + @Override + public boolean include(String key, EvidenceKeyFilter pipelineUses) { if(key.equalsIgnoreCase(EVIDENCE_SESSIONID) || key.equalsIgnoreCase(EVIDENCE_SEQUENCE)) { return false; } - return super.include(key); + return super.include(key, pipelineUses); } } diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/SequenceData.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/SequenceData.java index e6f18024d..6f90e3c4f 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/SequenceData.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/SequenceData.java @@ -52,6 +52,15 @@ public SequenceData(Logger logger, FlowData flowData) { super(logger, flowData); } + /** + * Holds only its values, so it is reused through its Flow Element's + * pool once the Flow Data that holds it is closed. + */ + @Override + protected boolean isPoolable() { + return true; + } + /** * Get the identifier of the session. * @return the session id diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/SetHeadersData.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/SetHeadersData.java index fb94c0fe1..92dc70fed 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/SetHeadersData.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/SetHeadersData.java @@ -52,6 +52,15 @@ public SetHeadersData( } //! [constructor] + /** + * Holds only its values, so it is reused through its Flow Element's + * pool once the Flow Data that holds it is closed. + */ + @Override + protected boolean isPoolable() { + return true; + } + /** * Get the response headers to be set. * @return table of response headers diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java index 48e1e1b88..5abad8ccc 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java @@ -58,6 +58,13 @@ public class Constants { fiftyone.pipeline.core.Constants.EVIDENCE_QUERY_PREFIX + fiftyone.pipeline.core.Constants.EVIDENCE_SEPERATOR + EVIDENCE_SEQUENCE_SUFIX; + + /** + * Whether a 51Degrees Pipeline builder adds the Usage Sharing Element, + * unless it is told otherwise. + */ + public static final boolean SHARE_USAGE_DEFAULT_ENABLED = true; + public static final int SHARE_USAGE_DEFAULT_REPEAT_EVIDENCE_MINUTES = 20; public static final double SHARE_USAGE_DEFAULT_SHARE_PERCENTAGE = 1.0; @@ -89,6 +96,19 @@ public class Constants { */ static final int SHARE_USAGE_DEFAULT_HTTP_POST_TIMEOUT = 5000; + /** + * The most time closing the Usage Sharing Element may take, from its + * start, in milliseconds. + */ + static final int SHARE_USAGE_CLOSE_LIMIT_MILLIS = 5000; + + /** + * The part of the close limit, in milliseconds, kept for what closing + * does after it stops waiting for the data to be sent: discarding the + * rest, logging the warning and freeing the resources. + */ + static final int SHARE_USAGE_CLOSE_RESERVE_MILLIS = 500; + /** * Default lost data count for share usage before it wraps */ @@ -105,6 +125,13 @@ public class Constants { */ static final String SHARE_USAGE_DEFAULT_URL = "https://devices-v4.51degrees.com/new.ashx"; + /** + * Deprecated. The default of the share all query string parameters + * option, which has no effect. + */ + public static final boolean + SHARE_USAGE_DEFAULT_SHARE_ALL_QUERY_STRING_PARAMETERS = true; + /** * Constant Added to blocked HTTP Headers so that the cookie header is * blocked by default. diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilder.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilder.java index dfe8a3e60..a111b1723 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilder.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilder.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.engines.fiftyone.flowelements; +import fiftyone.pipeline.annotations.DefaultValue; import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.PipelineBuilder; import fiftyone.pipeline.engines.services.HttpClientDefault; @@ -33,7 +34,7 @@ */ public class FiftyOnePipelineBuilder extends PipelineBuilder { - private boolean shareUsageEnabled = true; + private boolean shareUsageEnabled = Constants.SHARE_USAGE_DEFAULT_ENABLED; /** * Construct a new builder. @@ -57,6 +58,7 @@ public FiftyOnePipelineBuilder(ILoggerFactory loggerFactory) { * @param enabled true to enable usage sharing, false to disable * @return this builder */ + @DefaultValue(booleanValue = Constants.SHARE_USAGE_DEFAULT_ENABLED) public FiftyOnePipelineBuilder setShareUsage(boolean enabled) { this.shareUsageEnabled = enabled; return this; diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/PackageVersions.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/PackageVersions.java new file mode 100644 index 000000000..caadb2afc --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/PackageVersions.java @@ -0,0 +1,229 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; + +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.IOException; +import java.io.InputStream; +import java.net.JarURLConnection; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Enumeration; +import java.util.List; +import java.util.Properties; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +/** + * Finds the packages a Pipeline uses and the version of each, for the + * Package elements of the usage sharing record. A package's jar holds its + * Maven descriptor, being pom.properties, which gives the version, and + * pom.xml, which names the packages it depends on. A class that was not + * loaded from a jar, such as one compiled in the same project, gives no + * package. + */ +final class PackageVersions { + + /** + * The Maven group every 51Degrees package is published under. + */ + static final String GROUP_ID = "com.51degrees"; + + private PackageVersions() { + } + + /** + * The packages of the group that hold the classes, and every package of + * the group they depend on, each named "group:artifact" and given its + * version, in name order. + * @param classes the classes in use + * @param groupId the Maven group of the packages to find + * @return the packages and their versions + */ + static SortedMap of(Collection> classes, + String groupId) { + SortedMap packages = new TreeMap<>(); + for (Class type : classes) { + ClassLoader loader = type.getClassLoader(); + if (loader == null) { + continue; + } + for (String artifact : artifactsHolding(type, groupId)) { + add(artifact, groupId, loader, packages); + } + } + return packages; + } + + /** + * The artifacts of the group whose descriptor is in the jar the class + * was loaded from. A jar that bundles several packages gives each of + * them. + */ + static List artifactsHolding(Class type, String groupId) { + String path = type.getName().replace('.', '/') + ".class"; + URL url = type.getClassLoader().getResource(path); + if (url == null || "jar".equals(url.getProtocol()) == false) { + return Collections.emptyList(); + } + String folder = "META-INF/maven/" + groupId + "/"; + List artifacts = new ArrayList<>(); + try { + // A connection to the class's entry gives the whole jar. + URLConnection connection = url.openConnection(); + if (connection instanceof JarURLConnection == false) { + return artifacts; + } + // Without the cache the jar is opened for this read alone, so it + // is closed here and never held open for the application. + connection.setUseCaches(false); + try (JarFile jar = ((JarURLConnection) connection).getJarFile()) { + Enumeration entries = jar.entries(); + while (entries.hasMoreElements()) { + String name = entries.nextElement().getName(); + if (name.startsWith(folder) && + name.endsWith("/pom.properties")) { + String artifact = name.substring(folder.length(), + name.length() - "/pom.properties".length()); + if (artifact.indexOf('/') < 0) { + artifacts.add(artifact); + } + } + } + } + } catch (Exception e) { + // A jar that cannot be read gives no package. + } + return artifacts; + } + + private static void add(String artifact, + String groupId, + ClassLoader loader, + SortedMap packages) { + String name = groupId + ":" + artifact; + if (packages.containsKey(name)) { + return; + } + String folder = "META-INF/maven/" + groupId + "/" + artifact + "/"; + String version = version(loader.getResource(folder + "pom.properties")); + if (version == null) { + return; + } + packages.put(name, version); + for (String dependency : dependencies( + loader.getResource(folder + "pom.xml"), groupId)) { + add(dependency, groupId, loader, packages); + } + } + + private static String version(URL properties) { + if (properties == null) { + return null; + } + Properties values = new Properties(); + try (InputStream input = open(properties)) { + values.load(input); + } catch (Exception e) { + return null; + } + String version = values.getProperty("version"); + return version == null || version.trim().isEmpty() ? + null : version.trim(); + } + + /** + * The artifacts of the group that the pom names as dependencies, other + * than those only its tests use. A dependency in the pom's own group is + * often named with ${project.groupId}. + */ + static List dependencies(URL pom, String groupId) { + List artifacts = new ArrayList<>(); + if (pom == null) { + return artifacts; + } + try (InputStream input = open(pom)) { + DocumentBuilderFactory factory = + DocumentBuilderFactory.newInstance(); + factory.setFeature( + "http://apache.org/xml/features/disallow-doctype-decl", true); + factory.setExpandEntityReferences(false); + factory.setXIncludeAware(false); + Document document = factory.newDocumentBuilder().parse(input); + for (Element list : children(document.getDocumentElement(), + "dependencies")) { + for (Element dependency : children(list, "dependency")) { + String group = text(dependency, "groupId"); + String artifact = text(dependency, "artifactId"); + if (artifact != null && + "test".equals(text(dependency, "scope")) == false && + (groupId.equals(group) || + "${project.groupId}".equals(group) || + "${groupId}".equals(group))) { + artifacts.add(artifact); + } + } + } + } catch (Exception e) { + // A pom that cannot be read names no dependency. + } + return artifacts; + } + + /** + * Reads a resource without the JDK's cache of jars, so that the jar is + * closed with the stream. + */ + private static InputStream open(URL resource) throws IOException { + URLConnection connection = resource.openConnection(); + connection.setUseCaches(false); + return connection.getInputStream(); + } + + private static List children(Element parent, String name) { + List found = new ArrayList<>(); + for (Node node = parent.getFirstChild(); + node != null; + node = node.getNextSibling()) { + if (node instanceof Element && + name.equals(node.getNodeName())) { + found.add((Element) node); + } + } + return found; + } + + private static String text(Element parent, String name) { + List found = children(parent, name); + return found.isEmpty() ? null : found.get(0).getTextContent().trim(); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java index 588f1947c..1eda1efeb 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java @@ -38,6 +38,10 @@ /** * Sequence element establishes session and sequence evidence in the pipeline. + * As it adds Evidence, it must not run in a parallel group. Add it on its + * own, before the elements that read the session and sequence. The Pipeline + * does not check this. + * @see Specification, Evidence in a parallel group * @see Specification */ public class SequenceElement diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElement.java index 9bb05c6ea..d464152a7 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElement.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.Hashtable; import java.util.List; +import java.util.Locale; import java.util.Map; import org.slf4j.Logger; @@ -235,7 +236,7 @@ private PipelineConfig populateConfig(Pipeline pipeline) { PipelineConfig config = new PipelineConfig(); pipeline.getElementAvailableProperties().forEach((k, v) -> { v.forEach((p, m) -> { - if (p.toLowerCase() + if (p.toLowerCase(Locale.ROOT) .startsWith(SET_HEADER_PREFIX_LOWER_CASE)) { PropertyDetails details = new PropertyDetails(); details.propertyMetaData = m; diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index 1fd5bf088..e6c68ef46 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -26,6 +26,8 @@ import fiftyone.pipeline.core.data.*; import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.FlowElementBase; +import fiftyone.pipeline.core.flowelements.ParallelElements; +import fiftyone.pipeline.core.flowelements.RunsLast; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.configuration.CacheConfiguration; import fiftyone.pipeline.engines.fiftyone.data.EvidenceKeyFilterShareUsage; @@ -43,8 +45,13 @@ import java.net.SocketException; import java.net.UnknownHostException; import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.*; import java.util.concurrent.*; +import java.util.function.Function; +import java.util.function.LongSupplier; import static fiftyone.pipeline.core.Constants.EVIDENCE_SEPERATOR; import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.LOST_DATA_RESET_DEFAULT; @@ -53,17 +60,36 @@ /** * Abstract base class for ShareUsage elements. Contains common functionality * such as filtering the evidence and building the XML records. - * @see Specification + * @see Specification */ public abstract class ShareUsageBase - extends FlowElementBase { + extends FlowElementBase + implements RunsLast { /** * IP Addresses of local host device. */ private static final InetAddress[] localHosts; + + /** The form of DateSent, which is the time in UTC. */ + private static final String DATE_SENT_PATTERN = "yyyy-MM-dd'T'HH:mm:ss"; + + private static final DateTimeFormatter DATE_SENT_FORMAT = + DateTimeFormatter.ofPattern(DATE_SENT_PATTERN, Locale.ROOT) + .withZone(ZoneOffset.UTC); + + /** + * Formats a date as DateSent, in UTC. + * @deprecated the record takes DateSent from {@link #dateSentNow()}, + * which is safe to call from any thread. + */ + @Deprecated protected static final SimpleDateFormat DATE_FMT = - new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + new SimpleDateFormat(DATE_SENT_PATTERN); + + static { + DATE_FMT.setTimeZone(TimeZone.getTimeZone("UTC")); + } static { InetAddress[] localHosts1; @@ -117,7 +143,7 @@ public abstract class ShareUsageBase /** * Which of the values found are shared, leaving out the session cookie. */ - private final EvidenceKeyFilter evidenceKeyFilterExclSession; + private final EvidenceKeyFilterShareUsage evidenceKeyFilterExclSession; /** * The filter used to determine if an item of evidence should be ignored or @@ -184,8 +210,54 @@ public abstract class ShareUsageBase private List flowElements = null; /** - * Version of the OS the pipeline is being run on, as reported by - * System.getProperty("os.name"). + * The packages the pipeline uses, each with its version. This is + * populated when {@link #getPackages()} is called. + */ + private SortedMap packages = null; + + /** + * Finds the packages that hold the classes given and the packages they + * depend on. Tests replace it, as the classes they run are not loaded + * from published jars. + */ + Function>, SortedMap> packageFinder = + classes -> PackageVersions.of(classes, PackageVersions.GROUP_ID); + + /** + * How long after the invariant values were last written to a record + * they are written again, which is an hour. + */ + private static final long INVARIANT_VALUES_INTERVAL_MILLIS = + 60L * 60L * 1000L; + + /** + * The time in milliseconds, which decides when the invariant values are + * written again. Tests replace it, so that the hour passes without + * waiting for it. + */ + LongSupplier clock = System::currentTimeMillis; + + /** + * The process identifier, a version 4 UUID created when the element is + * built. It is never stored, so a process that starts again, or builds + * another element, has a new one. Written to every record as + * ProcessId. + */ + private final String processId = UUID.randomUUID().toString(); + + /** + * When the invariant values were last written to a record, by + * {@link #clock}, or null if they have not been written yet. + */ + private Long invariantValuesWritten = null; + + /** Guards {@link #invariantValuesWritten}. */ + private final Object invariantValuesLock = new Object(); + + /** + * The name of the operating system the Pipeline runs on, followed by + * its version where the JVM gives one, as System.getProperty("os.name") + * and System.getProperty("os.version") report them. Written as Platform. */ protected final String osVersion; @@ -218,6 +290,12 @@ public abstract class ShareUsageBase */ private volatile boolean stopping = false; + /** + * Set when closing has waited as long as it may for the queue to be + * sent. The data left is discarded and a consumer sends nothing more. + */ + private volatile boolean discarding = false; + /** * Keep a count of data lost */ @@ -434,7 +512,7 @@ protected ShareUsageBase( Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT, Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT, false, - false); + Constants.SHARE_USAGE_DEFAULT_SHARE_ALL_QUERY_STRING_PARAMETERS); } /** @@ -479,7 +557,8 @@ protected ShareUsageBase( * @param recoveryMultiplier what the pause is multiplied by each time the * send tried when a pause ends fails * @param shareAllEvidence if true then all Evidence is shared, apart from - * the keys that are never shared + * the keys that are never shared and the cookies + * other than the 51Degrees ones * @param shareAllQueryStringParameters if true then all query string * parameters are shared */ @@ -536,7 +615,8 @@ protected ShareUsageBase( // Some data is going to stay the same on all requests so we can // gather that now. languageVersion = System.getProperty("java.version"); - osVersion = System.getProperty("os.name"); + osVersion = platform( + System.getProperty("os.name"), System.getProperty("os.version")); this.enginesVersion = getClass().getPackage().getImplementationVersion(); coreVersion = Pipeline.class.getPackage().getImplementationVersion(); @@ -576,7 +656,60 @@ protected ShareUsageBase( } /** - * Return a list of {@link FlowElement} in the pipeline. + * The time now in UTC, in the form DateSent is written in. + * @return the text for DateSent + */ + protected static String dateSentNow() { + return DATE_SENT_FORMAT.format(Instant.now()); + } + + /** + * The process identifier, which every record carries as ProcessId. It + * is created when the element is built and is the same in every record + * the element writes. + * @return a version 4 UUID in its string form + */ + protected String getProcessId() { + return processId; + } + + /** + * Decides whether the record about to be written carries the invariant + * values: Version, Product, Flow Elements, Packages, Language, + * LanguageVersion, ServerIP and Platform. They are written to the first + * record and then to the first record written once an hour or more has + * passed since they were last written, and to no other record. Call it + * once for each record, as it notes the time when it returns true. + * @return true if the record is to carry the invariant values + */ + protected boolean writeInvariantValuesNow() { + long now = clock.getAsLong(); + synchronized (invariantValuesLock) { + if (invariantValuesWritten == null || + now - invariantValuesWritten >= + INVARIANT_VALUES_INTERVAL_MILLIS) { + invariantValuesWritten = now; + return true; + } + return false; + } + } + + /** + * The text for Platform, which is the name of the operating system + * followed by its version where there is one. + */ + private static String platform(String name, String version) { + String platform = name == null ? "" : name.trim(); + if (version != null && version.trim().isEmpty() == false) { + platform = (platform + " " + version.trim()).trim(); + } + return platform; + } + + /** + * Return the fully qualified class name of each {@link FlowElement} in + * the pipeline. * If the list is null then populate from the pipeline. * If there are multiple or no pipelines then log an error. * @return list of flow elements @@ -589,7 +722,7 @@ protected List getFlowElements() { pipeline = getPipelines().get(0); List list = new ArrayList<>(); for ( FlowElement element : pipeline.getFlowElements()) { - list.add(element.getClass().getSimpleName()); + list.add(element.getClass().getName()); } flowElements = list; } else { @@ -607,6 +740,49 @@ protected List getFlowElements() { return flowElements; } + /** + * The 51Degrees packages in use, each named by its Maven group and + * artifact id and given its version, in name order. They are the + * package that holds the pipeline, the package that holds each of its + * flow elements and this element, and every 51Degrees package those + * depend on. The pipeline lists a parallel group as one element, so the + * members of each group are added as well. + * @return the packages and their versions + */ + protected Map getPackages() { + if (packages == null) { + List> classes = new ArrayList<>(); + classes.add(getClass()); + if (getPipelines().size() == 1) { + Pipeline pipeline = getPipelines().get(0); + classes.add(pipeline.getClass()); + addElementClasses(classes, pipeline.getFlowElements()); + } + packages = packageFinder.apply(classes); + } + return packages; + } + + /** + * Add the class of each flow element, and of each member of a parallel + * group among them, to the classes in use. + * @param classes the classes in use + * @param elements the flow elements to add + */ + @SuppressWarnings("rawtypes") + private static void addElementClasses( + List> classes, + List elements) { + for (FlowElement element : elements) { + classes.add(element.getClass()); + if (element instanceof ParallelElements) { + addElementClasses( + classes, + ((ParallelElements) element).getFlowElements()); + } + } + } + @Override public String getElementDataKey() { return "shareusage"; @@ -707,6 +883,15 @@ protected boolean isStopping() { return stopping; } + /** + * Returns true once closing has stopped waiting for the queue to be + * sent. A consumer that sees this sends nothing more. + * @return true if the data left is being discarded + */ + protected boolean isDiscarding() { + return discarding; + } + /** * Returns true if there is a thread attempting to send usage data. * @return true if data is being sent @@ -730,8 +915,31 @@ public DataFactory getDataFactory() { return null; } + /** + * The time closing may still wait for the data to be sent, so that the + * whole of closing, measured from its start, keeps within the close + * limit with the reserve left for what follows the wait. + * @param closeStartNanos {@link System#nanoTime()} when closing started + * @return milliseconds left to wait, never less than 0 + */ + private static long closeWaitLeftMillis(long closeStartNanos) { + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis( + System.nanoTime() - closeStartNanos); + return Math.max(0, closeWaitMillis() - elapsedMillis); + } + + /** + * The longest closing waits for the data to be sent, from its start. + * @return milliseconds + */ + private static long closeWaitMillis() { + return Constants.SHARE_USAGE_CLOSE_LIMIT_MILLIS - + Constants.SHARE_USAGE_CLOSE_RESERVE_MILLIS; + } + @Override protected void managedResourcesCleanup() { + long closeStartNanos = System.nanoTime(); // Tell any consumer - one that is already running as well as the final // one queued below - that the queue has to be drained completely rather // than stopping once fewer than minEntriesPerMessage entries are left. @@ -750,8 +958,21 @@ protected void managedResourcesCleanup() { submitSendData(); try { executor.shutdown(); - if (!executor.awaitTermination(Constants.SHARE_USAGE_DEFAULT_HTTP_POST_TIMEOUT, TimeUnit.MILLISECONDS)){ - logger.warn("Could not send final data on close down"); + if (!executor.awaitTermination( + closeWaitLeftMillis(closeStartNanos), + TimeUnit.MILLISECONDS)) { + // Sending stops rather than hold up the process that closes + // the element, and the data left is discarded, all within + // the close limit. + discarding = true; + int discarded = + evidenceCollection.drainTo(new ArrayList()); + logger.warn( + "{} usage sharing entries were not sent, as sending had " + + "not finished {} milliseconds after usage sharing " + + "started closing.", + discarded, + closeWaitMillis()); executor.shutdownNow(); } } catch (InterruptedException interruptedException) { @@ -790,6 +1011,18 @@ private boolean isLocalHost(String address) throws UnknownHostException { return false; } + /** + * The Evidence key filter of the Pipeline processing the Flow Data, + * which decides which query string and form parameters it uses. + * @param data the Flow Data being processed + * @return the filter, or null if the Flow Data has no Pipeline + */ + private static EvidenceKeyFilter pipelineUses(FlowData data) { + return data.getPipeline() == null ? + null : + data.getPipeline().getEvidenceKeyFilter(); + } + /** * Process the supplied request data * @param data the {@link FlowData} instance that provides the evidence @@ -807,7 +1040,9 @@ private void processData(FlowData data, Map found) { // it to the collection. try { if (evidenceCollection.offer( - getDataFromEvidence(shared), + getDataFromEvidence( + shared, + pipelineUses(data)), addTimeout, TimeUnit.MILLISECONDS)) { queueFullReported = false; @@ -846,15 +1081,23 @@ private void processData(FlowData data, Map found) { * reference to the existing evidence instance. * * @param found the values {@link ShareUsageEvidence#find} found + * @param pipelineUses the Evidence key filter of the Pipeline, which + * decides which parameters it uses * @return a {@link ShareUsageData} instance populated with the values * the sharing rules allow */ - private ShareUsageData getDataFromEvidence(Map found) { + private ShareUsageData getDataFromEvidence( + Map found, + EvidenceKeyFilter pipelineUses) { ShareUsageData shareUsageData = new ShareUsageData(); for (Map.Entry entry : found.entrySet()) { - // Check if we can send this piece of evidence - if (evidenceKeyFilterExclSession.include(entry.getKey()) == false) { + // Check if we can send this piece of evidence. A query string or + // form parameter is sent only if it starts with 51D_ or the + // Pipeline uses it. + if (evidenceKeyFilterExclSession.include( + entry.getKey(), + pipelineUses) == false) { continue; } @@ -1030,4 +1273,15 @@ void tryAddToData(String key, Object value) { } } + + /** + * Usage sharing reads what every other Flow Element has added to the + * Evidence, so it is the last Flow Element of a Pipeline and outside any + * parallel group. + * @return the name a misplaced usage sharing element is refused with + */ + @Override + public String getRunsLastName() { + return "The usage sharing element"; + } } diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java index 6e07f6dd3..1b9822e3f 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java @@ -65,7 +65,8 @@ public abstract class ShareUsageBuilderBase { protected double maxRecoveryDelaySeconds = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT; protected double recoveryMultiplier = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT; protected boolean shareAllEvidence; - protected boolean shareAllQueryStringParameters; + protected boolean shareAllQueryStringParameters = + Constants.SHARE_USAGE_DEFAULT_SHARE_ALL_QUERY_STRING_PARAMETERS; /** * Constructor @@ -90,20 +91,17 @@ public ShareUsageBuilderBase(ILoggerFactory loggerFactory, Logger logger) { } /** - * By default, only query string parameters that start with 51D_ are - * shared. - *

    - * This setter replaces the current list, if any - *

    - * This setting allows you to share these parameters with 51Degrees - * if needed. + * Deprecated, and has no effect. A query string or HTTP form parameter + * is shared only when its name starts with 51D_ or it is Evidence the + * Pipeline uses, being a key a Flow Element in the Pipeline accepts, + * whatever this is set to. * @param queryStringParameterNames the (case-insensitive) names of * the query string parameters * to include * @return this builder */ - @DefaultValue("No sharing unless the parameter starts with 51D_") + @DefaultValue("None, as it has no effect") public ShareUsageBuilderBase setIncludedQueryStringParameters( List queryStringParameterNames) { includedQueryStringParameters.clear(); @@ -112,19 +110,16 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( } /** - * By default, only query string parameters that start with 51D_ are - * shared. - *

    - * This setter replaces the current list, if any - *

    - * This setting allows you to share these parameters with 51Degrees - * if needed. + * Deprecated, and has no effect. A query string or HTTP form parameter + * is shared only when its name starts with 51D_ or it is Evidence the + * Pipeline uses, being a key a Flow Element in the Pipeline accepts, + * whatever this is set to. * @param queryStringParameterNames a comma separated list of * (case-insensitive) names of the * query string parameters to include * @return this builder */ - @DefaultValue("No sharing unless the parameter starts with 51D_") + @DefaultValue("None, as it has no effect") public ShareUsageBuilderBase setIncludedQueryStringParameters( String queryStringParameterNames) { includedQueryStringParameters.clear(); @@ -134,16 +129,15 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( } /** - * By default, only query string parameters that start with 51D_ are - * shared. - *

    - * This setting adds the name of a parameter to share with 51Degrees - * if needed. + * Deprecated, and has no effect. A query string or HTTP form parameter + * is shared only when its name starts with 51D_ or it is Evidence the + * Pipeline uses, being a key a Flow Element in the Pipeline accepts, + * whatever this is set to. * @param queryStringParameterName the (case-insensitive) name of the * query string parameter to include * @return this builder */ - @DefaultValue("No sharing unless the parameter starts with 51D_") + @DefaultValue("None, as it has no effect") public ShareUsageBuilderBase setIncludedQueryStringParameter( String queryStringParameterName) { includedQueryStringParameters.add(queryStringParameterName.trim()); @@ -151,16 +145,14 @@ public ShareUsageBuilderBase setIncludedQueryStringParameter( } /** - * If set to true, every query string and HTTP form parameter is shared, - * rather than only those that start with 51D_ or are named in the - * included parameters. The whole query string and the parameters that are - * never shared are still not shared. - *

    - * Default value is false - * @param shareAll true to share all query string parameters + * Deprecated, and has no effect. Setting it shares no query string or + * HTTP form parameter that would not otherwise be shared, which is one + * whose name starts with 51D_ or that a Flow Element in the Pipeline + * accepts. + * @param shareAll kept, but not used * @return this builder */ - @DefaultValue(booleanValue = false) + @DefaultValue(booleanValue = true) public ShareUsageBuilderBase setShareAllQueryStringParameters( boolean shareAll) { this.shareAllQueryStringParameters = shareAll; @@ -168,8 +160,9 @@ public ShareUsageBuilderBase setShareAllQueryStringParameters( } /** - * Whether every query string parameter is shared. - * @return true if all query string parameters are shared + * Deprecated. The value set for the share all query string parameters + * option, which has no effect. + * @return the value set */ public boolean isShareAllQueryStringParameters() { return shareAllQueryStringParameters; @@ -177,10 +170,12 @@ public boolean isShareAllQueryStringParameters() { /** * If set to true, all Evidence is shared, which overrides the blocked - * HTTP headers and the rules for cookies, session values and query string - * parameters. The keys that are never shared (an email address, the salt - * for hashed email identifiers, the whole query string, the path and the - * Cookie header) are still not shared. + * HTTP headers and the rule for session values. The keys that are never + * shared (an email address, the salt for hashed email identifiers, the + * whole query string, the path and the Cookie header) are still not + * shared, and nor is any cookie whose name does not start with 51D_, or + * any query string or form parameter whose name does not start with 51D_ + * and which the Pipeline does not use. *

    * Default value is false * @param shareAll true to share all Evidence @@ -201,24 +196,22 @@ public boolean isShareAllEvidence() { } /** - * By default, all HTTP headers (excluding a few such as 'cookies', if they - * don't start with 51D_) are shared. Individual headers can be excluded from sharing by - * adding them to this list. + * By default, every HTTP header is shared except Cookie. Individual + * headers can be excluded from sharing by adding them to this list. *

    * This setter replaces the current list * @param blockedHeaders the (case-insensitive) names of the headers to block * @return this builder */ - @DefaultValue("All HTTP Headers are shared except cookies that do not start with 51D_") + @DefaultValue("All HTTP headers are shared except Cookie") public ShareUsageBuilderBase setBlockedHttpHeaders(List blockedHeaders) { blockedHttpHeaders = blockedHeaders; return this; } /** - * By default, all HTTP headers (excluding a few such as 'cookies', if they - * don't start with 51D_) are shared. Individual headers can be excluded from sharing by - * adding them to this list. + * By default, every HTTP header is shared except Cookie. Individual + * headers can be excluded from sharing by adding them to this list. *

    * This setter adds to the blocked headers * @param blockedHeaders a comma separated list of @@ -226,22 +219,21 @@ public ShareUsageBuilderBase setBlockedHttpHeaders(List blockedHeader * headers to include * @return this builder */ - @DefaultValue("All HTTP Headers are shared except cookies that do not start with 51D_") + @DefaultValue("All HTTP headers are shared except Cookie") public ShareUsageBuilderBase setBlockedHttpHeaders(String blockedHeaders) { blockedHttpHeaders.addAll(Arrays.asList(blockedHeaders.split("\\s*,\\s*"))); return this; } /** - * By default, all HTTP headers (excluding a few such as 'cookies', if they - * don't start with 51D_) are shared. Individual headers can be excluded from sharing by - * adding them to this list. + * By default, every HTTP header is shared except Cookie. Individual + * headers can be excluded from sharing by adding them to this list. *

    * This setter allows you to add the name of a header to block. * @param blockedHeader the (case-insensitive) name of the header to block * @return this builder */ - @DefaultValue("All HTTP Headers are shared except cookies that do not start with 51D_") + @DefaultValue("All HTTP headers are shared except Cookie") public ShareUsageBuilderBase setBlockedHttpHeader(String blockedHeader) { blockedHttpHeaders.add(blockedHeader.trim()); return this; @@ -358,6 +350,7 @@ public int getMaximumQueueSize() { * @return this builder */ @Deprecated + @DefaultValue(intValue = Constants.SHARE_USAGE_DEFAULT_ADD_TIMEOUT) public ShareUsageBuilderBase setAddTimeout(int milliseconds) { addTimeout = milliseconds; return this; @@ -388,6 +381,7 @@ public ShareUsageBuilderBase setAddTimeoutMillis(int milliseconds) { * @return this builder */ @Deprecated + @DefaultValue(intValue = Constants.SHARE_USAGE_DEFAULT_TAKE_TIMEOUT) public ShareUsageBuilderBase setTakeTimeout(int milliseconds) { takeTimeout = milliseconds; return this; @@ -450,7 +444,8 @@ public ShareUsageBuilderBase setSessionCookieName(String cookieName) { * If exactly the same evidence values are seen multiple times within this * time limit then they will only be shared once. *

    - * The default value for this is 0 - meaning always share. + * The default is 20 minutes. An interval of 0, or a negative one, + * shares every repeat. * @param interval the interval in minutes * @return this builder */ diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java index 58109cb2a..55aa9328b 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java @@ -43,6 +43,7 @@ import java.util.zip.GZIPOutputStream; import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_DEFAULT_HTTP_POST_TIMEOUT; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_DEFAULT_SHARE_ALL_QUERY_STRING_PARAMETERS; import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT; import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT; import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT; @@ -52,7 +53,7 @@ /** * Flow element that sends usage data to 51Degrees for analysis. - * @see Specification + * @see Specification */ public class ShareUsageElement extends ShareUsageBase { // max time to wait to send data @@ -332,7 +333,7 @@ public class ShareUsageElement extends ShareUsageBase { SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT, SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT, false, - false); + SHARE_USAGE_DEFAULT_SHARE_ALL_QUERY_STRING_PARAMETERS); } /** @@ -377,7 +378,9 @@ public class ShareUsageElement extends ShareUsageBase { * @param recoveryMultiplier what the pause is multiplied by each time the send * tried when a pause ends fails * @param shareAllEvidence if true then all Evidence is shared, apart from - * the keys that are never shared + * the keys that are never shared and + * the cookies other than the + * 51Degrees ones * @param shareAllQueryStringParameters if true then all query string * parameters are shared */ @@ -426,7 +429,7 @@ public class ShareUsageElement extends ShareUsageBase { shareAllEvidence, shareAllQueryStringParameters); - headers.put("Content-Type", "text/xml; charset=utf-8"); + headers.put("Content-Type", "text/xml"); // assume default GZIP connection headers.put("Content-Encoding", "gzip"); @@ -467,7 +470,7 @@ protected void sendUsageData() { // final consumer but the consumer that was already running has // drained everything - and there is no point posting a document // with no entries in it. - if (allData.isEmpty() == false) { + if (allData.isEmpty() == false && isDiscarding() == false) { boolean accepted = false; try { sendAsXML(allData); @@ -485,8 +488,9 @@ protected void sendUsageData() { logger.error("Exception sending usage data", exception); } // send in minEntries batches unless closing in which case drain the queue - } while (evidenceCollection.size() >= minEntriesPerMessage || - isStopping() && evidenceCollection.size() > 0); + } while (isDiscarding() == false && + (evidenceCollection.size() >= minEntriesPerMessage || + isStopping() && evidenceCollection.size() > 0)); logger.debug(threadMarker, "Stopping sending. Queue size is {}", evidenceCollection.size()); } @@ -521,7 +525,6 @@ protected void sendAsXML(List allData) throws Exception { private void streamXml(List allData, OutputStream os) throws XMLStreamException { XMLStreamWriter writer = xmlOutputFactory.createXMLStreamWriter(os, "UTF-8"); writer.writeStartElement("Devices"); - writer.writeAttribute("version", "1.1"); for (ShareUsageData data : allData) { writeXmlData(writer, data); } @@ -537,31 +540,51 @@ private void streamXml(List allData, OutputStream os) throws XML */ protected void writeXmlData(XMLStreamWriter writer, ShareUsageData data) throws XMLStreamException { writer.writeStartElement("Device"); - // The elements follow the order of the specification's example. + // The elements follow the order of the specification's example, and + // each one the example does not mark optional is always written, + // empty when there is no value. + // The values that do not change while the element lives are written + // to its first record, and again to the first record once an hour + // has passed since they were last written. The ProcessId ties the + // records that leave them out to them. + boolean invariant = writeInvariantValuesNow(); + // The identifier of this element, the same in every record + writeXmlElement(writer, "ProcessId", getProcessId()); // The SessionID used to track a series of requests - writeXmlElement(writer, "SessionId", data.sessionId); + writeRequiredXmlElement(writer, "SessionId", data.sessionId); // The sequence number of the request in a series of requests. writeXmlElement(writer, "Sequence", String.valueOf(data.sequence)); // The UTC date/time this entry was written - writeXmlElement(writer, "DateSent", DATE_FMT.format(new Date())); - // The version number of the Pipeline API - writeXmlElement(writer, "Version", coreVersion); - // Write Pipeline information - // The product name - writeXmlElement(writer, "Product", "Pipeline"); - // The flow elements in the current pipeline - for (String element : getFlowElements()) { - writeXmlElement(writer, "FlowElement", element); + writeXmlElement(writer, "DateSent", dateSentNow()); + if (invariant) { + // The version number of the Pipeline API + writeRequiredXmlElement(writer, "Version", coreVersion); + // Write Pipeline information + // The product name + writeXmlElement(writer, "Product", "Pipeline"); + // The flow elements in the current pipeline + for (String element : getFlowElements()) { + writeXmlElement(writer, "FlowElement", element); + } + // The version of every 51Degrees package in use + for (Map.Entry entry : getPackages().entrySet()) { + writer.writeStartElement("Package"); + writer.writeAttribute("Name", entry.getKey()); + writer.writeCharacters(entry.getValue()); + writer.writeEndElement(); + } + writeXmlElement(writer, "Language", "java"); + // The software language version + writeRequiredXmlElement(writer, "LanguageVersion", languageVersion); } - writeXmlElement(writer, "Language", "java"); - // The software language version - writeXmlElement(writer, "LanguageVersion", languageVersion); // The client IP of the request - writeXmlElement(writer, "ClientIP", data.clientIP); - // The IP of this server - writeXmlElement(writer, "ServerIP", getHostAddress()); - // The OS name and version - writeXmlElement(writer, "Platform", osVersion); + writeRequiredXmlElement(writer, "ClientIP", data.clientIP); + if (invariant) { + // The IP of this server + writeRequiredXmlElement(writer, "ServerIP", getHostAddress()); + // The OS name and version + writeRequiredXmlElement(writer, "Platform", osVersion); + } // Write all other evidence data that has been included. for (Map.Entry> category : data.evidenceData.entrySet()) { @@ -597,6 +620,20 @@ protected void writeXmlElement(XMLStreamWriter writer, } } + /** + * Write an element the record always holds, empty when there is no + * value. + */ + private void writeRequiredXmlElement( + XMLStreamWriter writer, + String elementName, + String elementContent) throws XMLStreamException { + writeXmlElement( + writer, + elementName, + elementContent == null ? "" : elementContent); + } + /** * replace characters that cause problems in XML with the "Replacement character" */ diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/interop/LibLoader.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/interop/LibLoader.java index 1492a34ba..65c97e55b 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/interop/LibLoader.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/interop/LibLoader.java @@ -28,6 +28,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.util.Locale; import java.util.Optional; /** @@ -44,7 +45,7 @@ public class LibLoader { * @throws UnsupportedOperationException if the OS is not supported */ private static OS getOs() throws UnsupportedOperationException { - String os = System.getProperty("os.name").toLowerCase(); + String os = System.getProperty("os.name").toLowerCase(Locale.ROOT); // Check for Windows if (os.contains("win")) { @@ -76,7 +77,7 @@ private static OS getOs() throws UnsupportedOperationException { * supported */ private static String getArch() throws UnsupportedOperationException { - String arch = System.getProperty("os.arch").toLowerCase(); + String arch = System.getProperty("os.arch").toLowerCase(Locale.ROOT); // Check aarch64/arm64 BEFORE generic arm (arm64 contains "arm") if (arch.contains("aarch64") || arch.contains("arm64")) { diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java index 8bfa52d3b..558daace3 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java @@ -28,6 +28,7 @@ import fiftyone.pipeline.core.data.EvidenceKeyFilter; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.engines.configuration.CacheConfiguration; +import fiftyone.pipeline.engines.fiftyone.data.EvidenceKeyFilterShareUsage; import fiftyone.pipeline.engines.fiftyone.data.ShareUsageEvidence; import fiftyone.pipeline.engines.trackers.TrackerBase; @@ -100,14 +101,24 @@ protected EvidenceKeyFilter getFilter() { * The key is made from the values {@link ShareUsageEvidence#find} finds * that the filter includes, which take in the web request's values, * as the session tracking hash is of the values that would be shared. - * The Evidence alone does not hold them. + * The Evidence alone does not hold them. A usage sharing filter is given + * the Pipeline's Evidence key filter, as the record is, so a query or + * form parameter the Pipeline uses is in the key. */ @Override protected DataKey generateKey(FlowData data) { + EvidenceKeyFilter pipelineUses = data.getPipeline() == null ? + null : + data.getPipeline().getEvidenceKeyFilter(); DataKeyBuilder builder = new DataKeyBuilderDefault(); for (Map.Entry entry : ShareUsageEvidence.find(data.getEvidence()).entrySet()) { - if (filter.include(entry.getKey())) { + boolean include = filter instanceof EvidenceKeyFilterShareUsage ? + ((EvidenceKeyFilterShareUsage) filter).include( + entry.getKey(), + pipelineUses) : + filter.include(entry.getKey()); + if (include) { builder.add( filter.order(entry.getKey()), entry.getKey(), diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java index 33eacefc6..0111b1931 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java @@ -22,6 +22,8 @@ package fiftyone.pipeline.engines.fiftyone.data; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; import org.junit.jupiter.api.Test; import java.util.Arrays; @@ -49,6 +51,10 @@ private static EvidenceKeyFilterShareUsage filter( DEFAULT_SESSION_COOKIE_NAME); } + private static EvidenceKeyFilter uses(String... keys) { + return new EvidenceKeyFilterWhitelist(Arrays.asList(keys)); + } + private static EvidenceKeyFilterShareUsage defaults() { return filter( Collections.emptyList(), @@ -56,11 +62,35 @@ private static EvidenceKeyFilterShareUsage defaults() { false); } + private static EvidenceKeyFilterShareUsage filter( + boolean includeSession, + boolean shareAllEvidence, + boolean shareAllQueryStringParameters) { + return new EvidenceKeyFilterShareUsage( + Collections.emptyList(), + Collections.emptyList(), + includeSession, + DEFAULT_SESSION_COOKIE_NAME, + shareAllEvidence, + shareAllQueryStringParameters); + } + /** - * By default headers other than Cookie are shared, cookies, session - * values and query string parameters only when their names start with - * 51D_, and any other evidence. + * Shares only the 51D_ query string parameters and those named, as when + * share all query string parameters is turned off. */ + private static EvidenceKeyFilterShareUsage namedParametersOnly( + List blockedHttpHeaders, + List includedQueryStringParams) { + return new EvidenceKeyFilterShareUsage( + blockedHttpHeaders, + includedQueryStringParams, + false, + DEFAULT_SESSION_COOKIE_NAME, + false, + false); + } + /** * The whole web request is read by the Usage Sharing Element and never * shared as a value, so the filter does not include it. @@ -73,6 +103,12 @@ public void webRequestIsNeverIncluded() { assertFalse(filter.include("Server.Http-Request")); } + /** + * By default every header but Cookie and any other evidence are shared, + * cookies and session values only when their names start with 51D_, and + * query string parameters only when their names start with 51D_ or the + * Pipeline uses them. + */ @Test public void defaultRules() { EvidenceKeyFilterShareUsage filter = defaults(); @@ -88,40 +124,120 @@ public void defaultRules() { assertTrue(filter.include("query.51d_gethighentropyvalues")); assertTrue(filter.include("query.51D_Test")); assertFalse(filter.include("query.password")); + assertFalse(filter.include("Query.Utm_Source")); + assertTrue(filter.include( + "query.utm_source", + uses("query.utm_source"))); assertFalse(filter.include("query.id.email")); + assertFalse(filter.include("server.request-query")); assertTrue(filter.include("server.client-ip")); assertTrue(filter.include("nodot")); } /** - * A name containing a dot is matched as a whole against the parameters to - * share. + * The deprecated included parameters list has no effect, so only the + * 51D_ parameters and those the Pipeline uses are shared. + */ + @Test + public void shareAllQueryStringParametersOff_OnlyNamedAnd51D() { + EvidenceKeyFilterShareUsage filter = namedParametersOnly( + Collections.emptyList(), + Collections.singletonList("campaign")); + + assertFalse(filter.include("query.password")); + assertFalse(filter.include("query.campaign")); + assertTrue(filter.include("query.campaign", uses("query.campaign"))); + assertTrue(filter.include("query.51D_Test")); + assertTrue(filter.include("header.user-agent")); + } + + /** + * Even when all Evidence is shared, a cookie is shared only when its + * name starts with 51D_, whatever its case. The session cookie is kept + * only by a filter that includes it for the tracker to compare. + */ + @Test + public void shareAllEvidence_NoCookieBut51D() { + EvidenceKeyFilterShareUsage sent = filter(false, true, true); + EvidenceKeyFilterShareUsage kept = filter(true, true, true); + + for (EvidenceKeyFilterShareUsage filter : Arrays.asList(sent, kept)) { + assertTrue(filter.include("cookie.51D_Profile")); + assertTrue(filter.include("cookie.51d_screenpixelswidth")); + assertFalse(filter.include("cookie._ga")); + assertFalse(filter.include("Cookie.Other")); + assertFalse(filter.include("cookie.a.b")); + assertTrue(filter.include("session.org.example.attribute")); + assertTrue(filter.include("header.x-anything")); + // Not even all Evidence shares a parameter the Pipeline does + // not use. + assertFalse(filter.include("query.password")); + assertTrue(filter.include( + "query.user-agent", + uses("query.user-agent"))); + } + assertFalse(sent.include("cookie." + DEFAULT_SESSION_COOKIE_NAME)); + assertTrue(kept.include("cookie." + DEFAULT_SESSION_COOKIE_NAME)); + } + + /** + * With all Evidence shared, the tracker compares the session cookie only + * when track session is on, and no other cookie but the 51D_ ones. + */ + @Test + public void trackerWithShareAllEvidence_SessionCookieOnlyWhenTracked() { + EvidenceKeyFilterShareUsageTracker tracked = + new EvidenceKeyFilterShareUsageTracker( + Collections.emptyList(), + Collections.emptyList(), + true, + DEFAULT_SESSION_COOKIE_NAME, + true, + true); + EvidenceKeyFilterShareUsageTracker untracked = + new EvidenceKeyFilterShareUsageTracker( + Collections.emptyList(), + Collections.emptyList(), + false, + DEFAULT_SESSION_COOKIE_NAME, + true, + true); + + assertTrue(tracked.include("cookie." + DEFAULT_SESSION_COOKIE_NAME)); + assertFalse(untracked.include("cookie." + DEFAULT_SESSION_COOKIE_NAME)); + assertFalse(tracked.include("cookie._ga")); + assertFalse(untracked.include("cookie._ga")); + assertTrue(untracked.include("cookie.51D_Profile")); + } + + /** + * A name containing a dot is matched as a whole against the keys the + * Pipeline uses. */ @Test public void fieldIsEverythingAfterTheFirstDot() { - EvidenceKeyFilterShareUsage filter = filter( + EvidenceKeyFilterShareUsage filter = namedParametersOnly( Collections.emptyList(), - Collections.singletonList("valid.email"), - false); + Collections.singletonList("valid.email")); + EvidenceKeyFilter uses = uses("query.valid.email"); - assertTrue(filter.include("query.valid.email")); - assertFalse(filter.include("query.valid")); - assertFalse(filter.include("query.other.name")); + assertTrue(filter.include("query.valid.email", uses)); + assertFalse(filter.include("query.valid", uses)); + assertFalse(filter.include("query.other.name", uses)); } /** - * The headers not to share and the parameters to share are matched - * without regard to case. + * The headers not to share are matched without regard to case. The + * deprecated parameters to share have no effect. */ @Test public void listsIgnoreCase() { - EvidenceKeyFilterShareUsage filter = filter( + EvidenceKeyFilterShareUsage filter = namedParametersOnly( Collections.singletonList("X-Secret"), - Collections.singletonList("Campaign"), - false); + Collections.singletonList("Campaign")); assertFalse(filter.include("header.x-secret")); - assertTrue(filter.include("query.campaign")); + assertFalse(filter.include("query.campaign")); assertTrue(filter.include("header.user-agent")); } @@ -187,6 +303,33 @@ public void cookieHeaderNeverShared() { assertTrue(filter.include("server.request-method")); } + /** + * By default every header but Cookie is shared, so a header named + * Cookies is shared like any other header. + */ + @Test + public void onlyTheCookieHeaderIsBlockedByDefault() { + EvidenceKeyFilterShareUsage filter = defaults(); + assertFalse(filter.include("header.cookie")); + assertFalse(filter.include("header.Cookie")); + assertTrue(filter.include("header.cookies")); + assertTrue(filter.include("header.Cookies")); + } + + /** + * A header named Cookies is blocked when the blocked headers name it. + */ + @Test + public void aCookiesHeaderIsBlockedWhenNamed() { + EvidenceKeyFilterShareUsage filter = filter( + Collections.singletonList("Cookies"), + Collections.emptyList(), + false); + assertFalse(filter.include("header.cookies")); + assertFalse(filter.include("header.Cookies")); + assertFalse(filter.include("header.cookie")); + } + /** * A key that only ends with the same letters as a key never shared is * treated as an ordinary key. @@ -198,7 +341,9 @@ public void neverSharedKeysMatchWholeParts() { Collections.singletonList("valid.email"), false); - assertTrue(filter.include("query.valid.email")); + assertTrue(filter.include( + "query.valid.email", + uses("query.valid.email"))); } /** @@ -212,8 +357,12 @@ public void namingAParameterDoesNotShareTheQueryString() { Collections.singletonList("utm_source"), false); - assertTrue(filter.include("query.utm_source")); - assertFalse(filter.include("server.request-query")); + assertTrue(filter.include( + "query.utm_source", + uses("query.utm_source"))); + assertFalse(filter.include( + "server.request-query", + uses("server.request-query"))); } /** diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/BuilderDefaultValueTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/BuilderDefaultValueTests.java new file mode 100644 index 000000000..8dab0cd98 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/BuilderDefaultValueTests.java @@ -0,0 +1,70 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.annotations.DefaultValue; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Every configuration setter of the usage sharing and 51Degrees Pipeline + * builders gives its default value beside it, with {@link DefaultValue}. + * See + * reference-implementation-notes.md line 79: + * "In all cases, there SHOULD be some mechanism to specify the default + * values alongside configuration property setters." + */ +public class BuilderDefaultValueTests { + + static List missingDefaults(Class... builders) { + List missing = new ArrayList<>(); + for (Class builder : builders) { + for (Method method : builder.getDeclaredMethods()) { + if (Modifier.isPublic(method.getModifiers()) && + method.getName().startsWith("set") && + method.isSynthetic() == false && + method.isBridge() == false && + method.getAnnotation(DefaultValue.class) == null) { + missing.add(builder.getSimpleName() + "#" + + method.getName() + + Arrays.toString(method.getParameterTypes())); + } + } + } + return missing; + } + + @Test + public void builderSetters_GiveTheirDefaults() { + assertEquals(new ArrayList(), missingDefaults( + ShareUsageBuilderBase.class, + FiftyOnePipelineBuilder.class)); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilderTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilderTests.java new file mode 100644 index 000000000..11c532c17 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOnePipelineBuilderTests.java @@ -0,0 +1,60 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Usage sharing is on unless the builder is told otherwise, as + * {@link Constants#SHARE_USAGE_DEFAULT_ENABLED} says. + */ +public class FiftyOnePipelineBuilderTests { + + @Test + public void ShareUsage_DefaultValue() { + assertTrue(Constants.SHARE_USAGE_DEFAULT_ENABLED); + } + + @SuppressWarnings("rawtypes") + @Test + public void ShareUsage_NotSet_FollowsDefault() throws Exception { + try (Pipeline pipeline = new FiftyOnePipelineBuilder() + .setAutoCloseElements(true) + .build()) { + int shareUsage = 0; + for (FlowElement element : pipeline.getFlowElements()) { + if (element instanceof ShareUsageElement) { + shareUsage++; + } + } + assertEquals( + Constants.SHARE_USAGE_DEFAULT_ENABLED ? 1 : 0, + shareUsage); + } + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/PackageVersionsTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/PackageVersionsTests.java new file mode 100644 index 000000000..f83b373e4 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/PackageVersionsTests.java @@ -0,0 +1,188 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The packages in use are found from the Maven descriptor in each jar, as + * the usage sharing record's Package elements need. + */ +public class PackageVersionsTests { + + /** + * A class to put in a jar, so that a test can load it from there. + */ + public static class InJar { + } + + @TempDir + Path folder; + + @Test + public void jar_FollowsTheGroupsDependencies() throws Exception { + Map a = withClass(); + descriptor(a, "a", "1.2.3", + "" + + dependency("com.51degrees", "e", null) + + "" + + dependency("${project.groupId}", "b", null) + + dependency("com.51degrees", "c", "test") + + dependency("org.example", "d", null) + + ""); + Map b = new LinkedHashMap<>(); + descriptor(b, "b", "4.5.6", ""); + Map c = new LinkedHashMap<>(); + descriptor(c, "c", "7.8.9", ""); + Map e = new LinkedHashMap<>(); + descriptor(e, "e", "1.0.0", ""); + + SortedMap expected = new TreeMap<>(); + expected.put("com.51degrees:a", "1.2.3"); + expected.put("com.51degrees:b", "4.5.6"); + assertEquals(expected, packagesOfClassIn( + write("a.jar", a), write("b.jar", b), write("c.jar", c), + write("e.jar", e))); + } + + @Test + public void jarHoldingSeveralPackages_GivesEachOfThem() throws Exception { + Map fat = withClass(); + descriptor(fat, "a", "1.0.0", ""); + descriptor(fat, "b", "2.0.0", ""); + + SortedMap expected = new TreeMap<>(); + expected.put("com.51degrees:a", "1.0.0"); + expected.put("com.51degrees:b", "2.0.0"); + assertEquals(expected, packagesOfClassIn(write("fat.jar", fat))); + } + + @Test + public void classFromAFolder_GivesNoPackage() { + assertEquals(Collections.emptyMap(), PackageVersions.of( + Collections.>singletonList(PackageVersionsTests.class), + PackageVersions.GROUP_ID)); + } + + /** + * A jar from Maven Central on the test's class path, so the reading is + * checked against a descriptor Maven wrote. + */ + @Test + public void publishedJar_GivesItsVersion() { + SortedMap packages = PackageVersions.of( + Collections.>singletonList(org.slf4j.Logger.class), + "org.slf4j"); + String version = packages.get("org.slf4j:slf4j-api"); + assertTrue(version != null && version.matches("\\d+\\.\\d+\\.\\d+.*"), + "slf4j-api should have a version, found " + packages); + } + + private SortedMap packagesOfClassIn(Path... jars) + throws Exception { + URL[] urls = new URL[jars.length]; + for (int i = 0; i < jars.length; i++) { + urls[i] = jars[i].toUri().toURL(); + } + try (URLClassLoader loader = new URLClassLoader(urls, null)) { + Class type = loader.loadClass(InJar.class.getName()); + assertEquals(loader, type.getClassLoader()); + return PackageVersions.of( + Collections.>singletonList(type), + PackageVersions.GROUP_ID); + } + } + + private static String dependency(String group, String artifact, + String scope) { + return "" + group + + "" + artifact + "" + + (scope == null ? "" : "" + scope + "") + + ""; + } + + private static void descriptor(Map entries, + String artifact, + String version, + String body) { + String path = "META-INF/maven/com.51degrees/" + artifact + "/"; + entries.put(path + "pom.properties", ("groupId=com.51degrees\n" + + "artifactId=" + artifact + "\nversion=" + version + "\n") + .getBytes(StandardCharsets.UTF_8)); + entries.put(path + "pom.xml", ("" + + "com.51degrees" + artifact + + "" + version + "" + body + + "").getBytes(StandardCharsets.UTF_8)); + } + + private static Map withClass() throws IOException { + String path = InJar.class.getName().replace('.', '/') + ".class"; + Map entries = new LinkedHashMap<>(); + try (InputStream input = PackageVersionsTests.class.getClassLoader() + .getResourceAsStream(path)) { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int read; + while ((read = input.read(buffer)) >= 0) { + bytes.write(buffer, 0, read); + } + entries.put(path, bytes.toByteArray()); + } + return entries; + } + + private Path write(String name, Map entries) + throws IOException { + Path file = folder.resolve(name); + try (OutputStream output = Files.newOutputStream(file); + JarOutputStream jar = new JarOutputStream(output)) { + for (Map.Entry entry : entries.entrySet()) { + jar.putNextEntry(new JarEntry(entry.getKey())); + jar.write(entry.getValue()); + jar.closeEntry(); + } + } + return file; + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/PooledElementDataTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/PooledElementDataTests.java new file mode 100644 index 000000000..977d30f38 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/PooledElementDataTests.java @@ -0,0 +1,74 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * The Sequence and Set Headers Element Data hold only their values, so they + * are pooled. Requests made one after another reuse one instance of each. + * @see Specification + */ +public class PooledElementDataTests { + + private static final int REQUESTS = 100; + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + @Test + public void oneThread_OneInstanceOfEach() throws Exception { + Set sequence = + Collections.newSetFromMap(new IdentityHashMap<>()); + Set headers = + Collections.newSetFromMap(new IdentityHashMap<>()); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement( + new SequenceElementBuilder(loggerFactory).build()) + .addFlowElement( + new SetHeadersElementBuilder(loggerFactory).build()) + .setAutoCloseElements(true) + .build()) { + for (int i = 0; i < REQUESTS; i++) { + try (FlowData data = pipeline.createFlowData()) { + data.process(); + sequence.add(data.get("sequence")); + headers.add(data.get("set-headers")); + } + } + } + assertEquals(1, sequence.size(), "Sequence Element Data allocated"); + assertEquals(1, headers.size(), "Set Headers Element Data allocated"); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElementParallelTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElementParallelTests.java new file mode 100644 index 000000000..ca6d361e8 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElementParallelTests.java @@ -0,0 +1,94 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.configuration.ElementOptions; +import fiftyone.pipeline.core.configuration.PipelineOptions; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * A Flow Element in a parallel group must not add Evidence, and that is the + * Flow Element's responsibility, so implementations do not check for it. A + * parallel group that holds the Sequence Element, which adds Evidence, is + * built by every public way of making one, as it always has been. + * @see Specification + */ +public class SequenceElementParallelTests { + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + private FlowElement sequence() { + return new SequenceElementBuilder(loggerFactory).build(); + } + + @Test + public void AddFlowElementsParallel_SequenceElement_IsBuilt() + throws Exception { + FlowElement[] group = + new FlowElement[] {sequence(), sequence()}; + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElementsParallel(group) + .setAutoCloseElements(true) + .build()) { + assertNotNull(pipeline); + } + } + + @Test + public void BuildFromConfiguration_SequenceElementInSubElements_IsBuilt() + throws Exception { + ElementOptions first = new ElementOptions(); + first.builderName = "SequenceElementBuilder"; + ElementOptions second = new ElementOptions(); + second.builderName = "SequenceElementBuilder"; + ElementOptions group = new ElementOptions(); + group.subElements = Arrays.asList(first, second); + PipelineOptions options = new PipelineOptions(); + options.elements.add(group); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .buildFromConfiguration(options)) { + assertNotNull(pipeline); + } + } + + @Test + public void AddFlowElement_SequenceElementOnItsOwn_IsBuilt() + throws Exception { + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(sequence()) + .setAutoCloseElements(true) + .build()) { + assertNotNull(pipeline); + } + } +} \ No newline at end of file diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBaseTest.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBaseTest.java index b91149c58..01a190e0a 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBaseTest.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBaseTest.java @@ -22,19 +22,19 @@ package fiftyone.pipeline.engines.fiftyone.flowelements; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.Arrays; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.*; public class ShareUsageBuilderBaseTest { ShareUsageBuilderBase base; - @Before + @BeforeEach public void setUp(){ base = new ShareUsageBuilderBase(null, null) { @Override @@ -78,9 +78,14 @@ public void testSetIncludedQueryStringParameters() { @Test public void setBlockedHttpHeaders() { + // The text setter adds to the list, as its Javadoc says, where the + // list setter replaces it. base.setBlockedHttpHeaders("g, h, i"); base.setBlockedHttpHeaders("a, b, c"); - assertEquals(3, base.blockedHttpHeaders.size()); + assertEquals(6, base.blockedHttpHeaders.size()); + assertTrue(base.blockedHttpHeaders.contains("g")); + assertTrue(base.blockedHttpHeaders.contains("h")); + assertTrue(base.blockedHttpHeaders.contains("i")); assertTrue(base.blockedHttpHeaders.contains("a")); assertTrue(base.blockedHttpHeaders.contains("b")); assertTrue(base.blockedHttpHeaders.contains("c")); @@ -111,22 +116,30 @@ public void testSetBlockedHttpHeaders() { } - @Test(expected = IllegalArgumentException.class) + @Test public void setIgnoreFlowDataEvidenceFilter1() { - base.setIgnoreFlowDataEvidenceFilter(""); + assertThrows(IllegalArgumentException.class, () -> { + base.setIgnoreFlowDataEvidenceFilter(""); + }); } - @Test(expected = IllegalArgumentException.class) + @Test public void setIgnoreFlowDataEvidenceFilter2() { - base.setIgnoreFlowDataEvidenceFilter("hello"); + assertThrows(IllegalArgumentException.class, () -> { + base.setIgnoreFlowDataEvidenceFilter("hello"); + }); } - @Test(expected = IllegalArgumentException.class) + @Test public void setIgnoreFlowDataEvidenceFilter2a() { - base.setIgnoreFlowDataEvidenceFilter("hello, one: two"); + assertThrows(IllegalArgumentException.class, () -> { + base.setIgnoreFlowDataEvidenceFilter("hello, one: two"); + }); } - @Test(expected = IllegalArgumentException.class) + @Test public void setIgnoreFlowDataEvidenceFilter2b() { - base.setIgnoreFlowDataEvidenceFilter("hello : world, one"); + assertThrows(IllegalArgumentException.class, () -> { + base.setIgnoreFlowDataEvidenceFilter("hello : world, one"); + }); } @Test public void setIgnoreFlowDataEvidenceFilter3() { @@ -143,13 +156,17 @@ public void setSharePercentage1() { base.setSharePercentage(0.1); assertEquals(0.1, base.sharePercentage, 0.00001); } - @Test(expected = IllegalArgumentException.class) + @Test public void setSharePercentage2() { - base.setSharePercentage(-0.1); + assertThrows(IllegalArgumentException.class, () -> { + base.setSharePercentage(-0.1); + }); } - @Test(expected = IllegalArgumentException.class) + @Test public void setSharePercentage3() { - base.setSharePercentage(1.1); + assertThrows(IllegalArgumentException.class, () -> { + base.setSharePercentage(1.1); + }); } @Test @@ -157,26 +174,34 @@ public void setMinimumEntriesPerMessage1() { base.setMinimumEntriesPerMessage(50); assertEquals(50, base.minimumEntriesPerMessage); } - @Test(expected = IllegalArgumentException.class) + @Test public void setMinimumEntriesPerMessage2() { - base.setMinimumEntriesPerMessage(0); + assertThrows(IllegalArgumentException.class, () -> { + base.setMinimumEntriesPerMessage(0); + }); } - @Test(expected = IllegalArgumentException.class) + @Test public void setMinimumEntriesPerMessage3() { - base.setMinimumEntriesPerMessage(-1); + assertThrows(IllegalArgumentException.class, () -> { + base.setMinimumEntriesPerMessage(-1); + }); } @Test public void setMaximumQueueSize() { base.setMaximumQueueSize(200); assertEquals(200, base.maximumQueueSize); } - @Test(expected = IllegalArgumentException.class) + @Test public void setMaximumQueueSize1() { - base.setMaximumQueueSize(0); + assertThrows(IllegalArgumentException.class, () -> { + base.setMaximumQueueSize(0); + }); } - @Test(expected = IllegalArgumentException.class) + @Test public void setMaximumQueueSize2() { - base.setMaximumQueueSize(-1); + assertThrows(IllegalArgumentException.class, () -> { + base.setMaximumQueueSize(-1); + }); } @Test @@ -190,9 +215,11 @@ public void setAddTimeoutMillis1() { base.setAddTimeoutMillis(1000); assertEquals(1000, base.addTimeout); } - @Test(expected = IllegalArgumentException.class) + @Test public void setAddTimeoutMillis2() { - base.setAddTimeoutMillis(-1); + assertThrows(IllegalArgumentException.class, () -> { + base.setAddTimeoutMillis(-1); + }); } @Test @@ -200,20 +227,24 @@ public void setTakeTimeoutMillis1() { base.setTakeTimeoutMillis(1000); assertEquals(1000, base.takeTimeout); } - @Test(expected = IllegalArgumentException.class) + @Test public void setTakeTimeoutMillis2() { - base.setTakeTimeoutMillis(-1); + assertThrows(IllegalArgumentException.class, () -> { + base.setTakeTimeoutMillis(-1); + }); } @Test public void setShareUsageUrl() { base.setShareUsageUrl("https://dog-food.com"); - assertEquals(base.shareUsageUrl, "https://dog-food.com"); + assertEquals("https://dog-food.com", base.shareUsageUrl); } - @Test(expected = IllegalArgumentException.class) + @Test public void setShareUsageUrl1() { - base.setShareUsageUrl("dog food"); + assertThrows(IllegalArgumentException.class, () -> { + base.setShareUsageUrl("dog food"); + }); } @Test @@ -221,11 +252,12 @@ public void setSessionCookieName() { base.setSessionCookieName("test"); assertEquals("test", base.sessionCookieName); } - @Test (expected = IllegalArgumentException.class) + @Test public void setSessionCookieName1() { - // illegal name - base.setSessionCookieName("te st"); - assertEquals("test", base.sessionCookieName); + assertThrows(IllegalArgumentException.class, () -> { + // illegal name + base.setSessionCookieName("te st"); + }); } @Test @@ -233,9 +265,12 @@ public void setRepeatEvidenceIntervalMinutes() { base.setRepeatEvidenceIntervalMinutes(1); assertEquals(1, base.repeatEvidenceInterval); } - @Test (expected = IllegalArgumentException.class) - public void setRepeatEvidenceIntervalMinutes1() { + @Test + public void setRepeatEvidenceIntervalMinutes_Negative_Kept() { + // A negative interval is not refused. It shares every repeat, as 0 + // does. base.setRepeatEvidenceIntervalMinutes(-1); + assertEquals(-1, base.repeatEvidenceInterval); } @Test diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageCloseTimeoutTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageCloseTimeoutTests.java new file mode 100644 index 000000000..a232bcd25 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageCloseTimeoutTests.java @@ -0,0 +1,275 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.DataUploader; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.lang.reflect.Proxy; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static fiftyone.pipeline.core.Constants.EVIDENCE_CLIENTIP_KEY; +import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Closing finishes within 5 seconds of its start. If the data waiting has + * not all been sent in time, sending stops, the rest is discarded and a + * warning gives the number discarded. See Cleanup in the usage sharing + * element specification. + */ +public class ShareUsageCloseTimeoutTests { + + /** + * The longest the test waits for something it expects to happen. + */ + private static final long WAIT_LIMIT_MILLIS = 30000; + + /** + * Entries processed, enough for several messages of at most two. + */ + private static final int ENTRIES = 10; + + /** + * The 5 seconds the specification bounds the whole of cleanup at. The + * usage + * sharing element specification says "Cleanup SHOULD complete within + * 5 seconds, so that closing the element does not hold up the process + * that closes it." + *

    + * The limit is the specification's, not a measurement. Closing must take + * less, holding the wait for the data to be sent, discarding the rest, + * the warning and freeing the resources. The element waits 4,500 ms, its + * 5,000 ms budget less a 500 ms reserve, then does the rest, so it + * closes in a little over 4,500 ms. Code that waits the full 5 seconds + * before doing the rest always takes at least this long, so it fails. + */ + private static final long CLOSE_LIMIT_MILLIS = 5000; + + /** + * An upload whose first answer does not come until the test releases + * it, as with a remote service that has stopped answering. Interrupting + * the waiting thread does not end the wait, as it would not end a + * stalled connection. + */ + private static class StalledUploader implements DataUploader { + private final CountDownLatch sending = new CountDownLatch(1); + private final CountDownLatch release = new CountDownLatch(1); + private final AtomicInteger sends = new AtomicInteger(); + + @Override + public OutputStream getOutputStream() { + return new ByteArrayOutputStream(); + } + + @Override + public int getResponseCode() { + sends.incrementAndGet(); + sending.countDown(); + while (release.getCount() > 0) { + try { + release.await(); + } catch (InterruptedException ignored) { + // Keeps waiting, as a stalled connection would. + } + } + return 200; + } + + @Override + public String getResponseContent() { + return ""; + } + } + + @Test + public void ShareUsageElement_CloseTimesOut_StopsSendingAndDiscards() + throws Exception { + List warnings = new CopyOnWriteArrayList<>(); + StalledUploader uploader = new StalledUploader(); + Pipeline pipeline = new PipelineBuilder().build(); + ShareUsageElement element = element(warnings, pipeline, uploader); + try { + for (int i = 1; i <= ENTRIES; i++) { + process(pipeline, element, "10.0.0." + i); + } + assertTrue( + uploader.sending.await(WAIT_LIMIT_MILLIS, TimeUnit.MILLISECONDS), + "No send was started."); + + element.close(); + uploader.release.countDown(); + + assertTrue( + element.executor.awaitTermination( + WAIT_LIMIT_MILLIS, + TimeUnit.MILLISECONDS), + "Sending did not stop after closing."); + assertEquals(1, uploader.sends.get(), + "Data was sent after closing gave up waiting."); + assertTrue( + warnings.stream().anyMatch(warning -> + warning.contains("were not sent") && + warning.matches(".*\\b[1-9][0-9]*\\b.*")), + "Discarding the data left was not logged with its count. " + + "Warnings: " + warnings); + } + finally { + uploader.release.countDown(); + if (element.executor.isShutdown() == false) { + element.close(); + } + pipeline.close(); + } + } + + /** + * Closing while the remote service has stopped answering finishes within + * 5 seconds of its start, the whole of cleanup included, while the + * upload in progress is still waiting for its answer. + */ + @Test + public void ShareUsageElement_ServiceStopsAnswering_CloseWithin5Seconds() + throws Exception { + StalledUploader uploader = new StalledUploader(); + Pipeline pipeline = new PipelineBuilder().build(); + ShareUsageElement element = element( + new CopyOnWriteArrayList<>(), + pipeline, + uploader); + try { + for (int i = 1; i <= ENTRIES; i++) { + process(pipeline, element, "10.0.0." + i); + } + assertTrue( + uploader.sending.await(WAIT_LIMIT_MILLIS, TimeUnit.MILLISECONDS), + "No send was started."); + + long[] tookMillis = new long[1]; + assertTimeoutPreemptively( + Duration.ofMillis(WAIT_LIMIT_MILLIS), + () -> { + long start = System.nanoTime(); + element.close(); + tookMillis[0] = TimeUnit.NANOSECONDS.toMillis( + System.nanoTime() - start); + }, + "Closing waited for the remote service to answer."); + assertTrue(tookMillis[0] < CLOSE_LIMIT_MILLIS, + "Closing took " + tookMillis[0] + " ms, not within the " + + CLOSE_LIMIT_MILLIS + " ms the specification puts on " + + "cleanup."); + assertEquals(1, uploader.release.getCount(), + "The upload in progress was answered before closing ended."); + } + finally { + uploader.release.countDown(); + if (element.executor.isShutdown() == false) { + element.close(); + } + pipeline.close(); + } + } + + /** + * An element that sends each entry on its own, through the uploader + * given, logging its warnings to the list given. + */ + private static ShareUsageElement element( + List warnings, + Pipeline pipeline, + DataUploader uploader) { + ShareUsageElement element = new ShareUsageElement( + recordingLogger(warnings), + 1, + 1, + 100, + 100, + 100, + 1, + true, + "http://localhost/usage", + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + DEFAULT_SESSION_COOKIE_NAME, + flowData -> true); + element.addPipeline(pipeline); + element.dataUploader = uploader; + return element; + } + + private static void process( + Pipeline pipeline, + ShareUsageElement element, + String ip) throws Exception { + Map evidence = new HashMap<>(); + evidence.put(EVIDENCE_CLIENTIP_KEY, ip); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidence); + element.process(data); + } + } + + /** + * A logger that records the warnings it is given. + */ + private static Logger recordingLogger(List warnings) { + return (Logger) Proxy.newProxyInstance( + Logger.class.getClassLoader(), + new Class[] {Logger.class}, + (proxy, method, args) -> { + if (method.getName().equals("warn")) { + warnings.add(Arrays.deepToString(args)); + return null; + } + if (method.getReturnType() == boolean.class) { + return false; + } + if (method.getReturnType() == int.class) { + return 0; + } + if (method.getReturnType() == String.class) { + return "test"; + } + return null; + }); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java index 0a6405558..618291e26 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java @@ -40,7 +40,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.time.Duration; import java.util.List; +import java.util.Locale; import java.util.concurrent.CopyOnWriteArrayList; import java.util.Map; import java.util.function.BooleanSupplier; @@ -52,6 +54,7 @@ import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -433,4 +436,148 @@ public void ShareUsageElement_FailedAnswer_ReportsStatusAndContent() { assertTrue(failure.getMessage().contains("500")); assertTrue(failure.getMessage().contains("the service says no")); } + + /** + * A logger that records each error, warning and information message it + * is given as its level followed by its arguments. + */ + private static Logger levelLogger(List records) { + return (Logger) Proxy.newProxyInstance( + Logger.class.getClassLoader(), + new Class[] {Logger.class}, + (proxy, method, args) -> { + String name = method.getName(); + if (name.equals("error") || + name.equals("warn") || + name.equals("info")) { + records.add(name.toUpperCase(Locale.ROOT) + " " + + Arrays.deepToString(args)); + return null; + } + if (method.getReturnType() == boolean.class) { + return false; + } + if (method.getReturnType() == int.class) { + return 0; + } + if (method.getReturnType() == String.class) { + return "test"; + } + return null; + }); + } + + /** + * A refused send is logged at Error with the status code and the + * content of the answer, and processing goes on, so evidence processed + * after it is still sent once the service answers. + */ + @Test + public void ShareUsageElement_RefusedSend_LoggedAndProcessingGoesOn() + throws Exception { + List records = new CopyOnWriteArrayList<>(); + logger = levelLogger(records); + useBackOff(DEFAULT_FAILURES, 2, 2, 100); + uploader.content = "the service says no"; + + failSends(1); + + assertTrue(records.stream().anyMatch(record -> + record.startsWith("ERROR") && + record.contains("500") && + record.contains("the service says no")), + "The refused send was not logged with its status and content. " + + "Logged: " + records); + uploader.code = 200; + String later = process(); + waitFor( + () -> uploader.accepted().contains(later), + () -> "Evidence processed after the refused send was not sent."); + } + + /** + * Pausing, and each lengthening of the pause, is logged as a warning + * that gives the delay, and resuming is logged as information. + */ + @Test + public void ShareUsageElement_PauseAndResume_AreLogged() throws Exception { + List records = new CopyOnWriteArrayList<>(); + logger = levelLogger(records); + useBackOff(1, 2, 8, 100); + + element.recordSendResult(false); + element.recordSendResult(false); + element.recordSendResult(true); + + int paused = indexOf(records, "WARN", "paused for", "2.0]"); + int lengthened = indexOf(records, "WARN", "paused for", "4.0]"); + int resumed = indexOf(records, "INFO", "resumed", "]"); + assertTrue(paused >= 0 && lengthened > paused && resumed > lengthened, + "The pause, its lengthening and the resumption were not logged " + + "in turn. Logged: " + records); + } + + /** + * Closing while paused discards what is queued and logs how many + * entries were not sent. + */ + @Test + public void ShareUsageElement_CloseWhilePaused_LogsTheNumberNotSent() + throws Exception { + List records = new CopyOnWriteArrayList<>(); + logger = levelLogger(records); + useBackOff(1, 60, 60, 100); + failSends(1); + process(); + process(); + process(); + + element.close(); + + assertTrue(indexOf(records, "", "were not sent", ", 3]") >= 0, + "The three entries not sent were not counted in the log. " + + "Logged: " + records); + } + + /** + * Processing during a pause neither waits for the pause nor is stopped + * by it, so the evidence is queued. The pause is two minutes, so + * processing that waited for it could not finish within the limit. + */ + @Test + public void ShareUsageElement_ProcessDuringPause_DoesNotWait() + throws Exception { + useBackOff(1, 120, 120, 100); + failSends(1); + + assertTimeoutPreemptively(Duration.ofSeconds(30), () -> { + for (int i = 0; i < 5; i++) { + process(); + } + }); + + assertEquals(1, uploader.count(), "A send was made during the pause."); + assertEquals(5, element.evidenceCollection.size(), + "The evidence processed during the pause was not queued."); + } + + /** + * The index of the first record at the level that holds the text and + * ends with the given end, or -1 if there is none. + */ + private static int indexOf( + List records, + String level, + String text, + String end) { + for (int i = 0; i < records.size(); i++) { + String record = records.get(i); + if (record.startsWith(level) && + record.contains(text) && + record.endsWith(end)) { + return i; + } + } + return -1; + } } diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java index a7ecfecbe..8526c9986 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java @@ -25,7 +25,13 @@ import fiftyone.common.testhelpers.LogbackHelper; import fiftyone.common.testhelpers.TestLogger; import fiftyone.common.testhelpers.TestLoggerFactory; +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.FlowElementBase; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.core.flowelements.PipelineBuilder; import fiftyone.pipeline.engines.services.DataUploader; @@ -171,6 +177,12 @@ public boolean track(FlowData flowData) { // Dependencies private Pipeline pipeline; + + /** + * A Pipeline holding the Sequence Element, which accepts the session id + * and sequence, so that usage sharing shares them. + */ + private Pipeline sequencePipeline; private Tracker tracker; TestConnector connector; @@ -197,6 +209,18 @@ public void cleanUp() throws Exception { shareUsageElement.close(); } pipeline.close(); + if (sequencePipeline != null) { + sequencePipeline.close(); + } + } + + private Pipeline sequencePipeline() throws Exception { + if (sequencePipeline == null) { + sequencePipeline = new PipelineBuilder() + .addFlowElement(sequenceElement) + .build(); + } + return sequencePipeline; } private void createShareUsage( @@ -314,6 +338,173 @@ private void copy(InputStream source, OutputStream target) throws IOException { } } + /** + * The record carries a Package element for each 51Degrees package in + * use, named by its Maven group and artifact id, with its version, after + * the FlowElement elements and before Language, as the specification's + * Processing section sets out. The packages are asked for with this + * element's class among the classes in use. + */ + @Test + public void ShareUsageElement_Packages_EachWithItsVersion() throws Exception { + createShareUsage( + 1, + 1, + 1, + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>()); + shareUsageElement.httpClient = new TestHttpClient(); + shareUsageElement.dataUploader = null; + SortedMap packages = new TreeMap<>(); + packages.put("com.51degrees:pipeline.core", "4.5.131"); + packages.put("com.51degrees:pipeline.engines.fiftyone", "4.5.132"); + List>> asked = new ArrayList<>(); + shareUsageElement.packageFinder = classes -> { + asked.add(classes); + return packages; + }; + + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); + shareUsageElement.process(data); + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] zip = ((TestHttpClient)shareUsageElement.httpClient).getBuffer(); + copy(new GZIPInputStream(new ByteArrayInputStream(zip)), baos); + Document doc = docBuilderFactory.newDocumentBuilder().parse( + new ByteArrayInputStream(baos.toByteArray())); + + NodeList found = (NodeList) xpath.compile("//Devices/Device/Package") + .evaluate(doc, XPathConstants.NODESET); + assertEquals(2, found.getLength(), "One Package element for each package"); + int i = 0; + for (Map.Entry entry : packages.entrySet()) { + org.w3c.dom.Element element = (org.w3c.dom.Element) found.item(i++); + assertEquals(entry.getKey(), element.getAttribute("Name")); + assertEquals(entry.getValue(), element.getTextContent()); + } + assertEquals("Product", previousElement(found.item(0)).getNodeName(), + "With no Flow Elements the Package elements follow Product"); + assertEquals("Language", nextElement(found.item(1)).getNodeName(), + "Language follows the Package elements"); + } + assertEquals(1, asked.size(), "The packages are found once"); + assertTrue(asked.get(0).contains(ShareUsageElement.class), + "This element's own package is among those in use"); + } + + /** + * A Flow Element of the test's own, which processes nothing. Each + * subclass is a class of its own, so the classes the packages are asked + * for show which elements were found. + */ + private static class GroupMember + extends FlowElementBase { + private final String key; + + GroupMember(String key) { + super(LoggerFactory.getLogger(GroupMember.class), null); + this.key = key; + } + + @Override + protected void processInternal(FlowData data) { + } + + @Override + public String getElementDataKey() { + return key; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist(new ArrayList()); + } + + @Override + public List getProperties() { + return Collections.emptyList(); + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + } + + private static class FirstGroupMember extends GroupMember { + FirstGroupMember() { + super("first"); + } + } + + private static class SecondGroupMember extends GroupMember { + SecondGroupMember() { + super("second"); + } + } + + /** + * A Flow Element in a parallel group is in use as much as one the + * Pipeline lists itself, so the packages are asked for with its class + * too. The specification's Package element is written "for every + * 51Degrees package in use, being the package that holds the Pipeline, + * the package that holds each FlowElement in the Pipeline, and every + * 51Degrees package those depend on". + */ + @Test + @SuppressWarnings("rawtypes") + public void ShareUsageElement_Packages_IncludeParallelGroupMembers() throws Exception { + pipeline.close(); + pipeline = new PipelineBuilder() + .addFlowElementsParallel(new FlowElement[] { + new FirstGroupMember(), + new SecondGroupMember() }) + .build(); + createShareUsage( + 1, + 1, + 1, + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>()); + List>> asked = new ArrayList<>(); + shareUsageElement.packageFinder = classes -> { + asked.add(new ArrayList<>(classes)); + return new TreeMap<>(); + }; + + shareUsageElement.getPackages(); + + assertEquals(1, asked.size(), "The packages are found once"); + assertTrue(asked.get(0).contains(FirstGroupMember.class), + "The first member of the group is among the classes in use"); + assertTrue(asked.get(0).contains(SecondGroupMember.class), + "The second member of the group is among the classes in use"); + } + + private static Node previousElement(Node node) { + Node sibling = node.getPreviousSibling(); + while (sibling != null && sibling.getNodeType() != Node.ELEMENT_NODE) { + sibling = sibling.getPreviousSibling(); + } + return sibling; + } + + private static Node nextElement(Node node) { + Node sibling = node.getNextSibling(); + while (sibling != null && sibling.getNodeType() != Node.ELEMENT_NODE) { + sibling = sibling.getNextSibling(); + } + return sibling; + } + // duplicate above test, only using legacy HttpClient approach @Test public void ShareUsageElement_SingleEvent_ClientIPAndHeader_HttpClient() throws Exception { @@ -514,7 +705,7 @@ public void ShareUsageElement_WebRequest_SharesWhatNoOtherElementAccepts() request.put("header.X-Blocked", "blocked"); request.put("cookie.51D_Profile", "123456"); request.put("cookie.other", "not-shared"); - request.put("query.secret", "not-shared"); + request.put("query.campaign", "shared-query"); request.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); request.put("server.request-method", "GET"); request.put("server.request-path", "/a%2Fb/c"); @@ -554,9 +745,14 @@ public void ShareUsageElement_WebRequest_SharesWhatNoOtherElementAccepts() "//cookie[@Name='51D_Profile']", doc), "A 51D_ cookie from the web request was not shared."); assertEquals(0, ((NodeList) xpath.evaluate( - "//cookie[@Name='other'] | //query[@Name='secret']", doc, + "//cookie[@Name='other']", doc, + XPathConstants.NODESET)).getLength(), + "A cookie the rules leave out was shared."); + assertEquals(0, ((NodeList) xpath.evaluate( + "//query[@Name='campaign']", doc, XPathConstants.NODESET)).getLength(), - "A cookie or query string parameter the rules leave out was shared."); + "A query string parameter that no Flow Element accepts " + + "was shared."); assertEquals("1.2.3.4", xpath.evaluate("//ClientIP", doc), "The client IP from the web request was not shared."); assertEquals("GET", xpath.evaluate( @@ -861,20 +1057,17 @@ public void ShareUsageElement_RecordFollowsTheSpecificationExample() names.add(child.getNodeName()); } } - String[] expected = {"SessionId", "Sequence", "DateSent", "Version", + String[] expected = {"ProcessId", "SessionId", "Sequence", "DateSent", "Version", "Product", "FlowElement", "Language", "LanguageVersion", "ClientIP", "ServerIP", "Platform"}; - List required = Arrays.asList("SessionId", "Sequence", - "DateSent", "Product", "Language", "LanguageVersion"); + // Every element is always written, empty when there is no value. + // FlowElement is written once for each element of the Pipeline, so + // not at all here, where the test's Pipeline holds no elements. int last = -1; for (String name : expected) { int index = names.indexOf(name); - if (!required.contains(name)) { - // Only present when the Pipeline, the request and the - // platform give one. - if (index < 0) { - continue; - } + if (index < 0 && name.equals("FlowElement")) { + continue; } assertTrue(index >= 0, name + " missing from " + names); assertTrue(index > last, name + " out of order in " + names); @@ -920,7 +1113,7 @@ public void ShareUsageElement_SessionIdAndSequence_None() throws Exception { Map evidenceData = new HashMap<>(); evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); - try (FlowData data = pipeline.createFlowData()) { + try (FlowData data = sequencePipeline().createFlowData()) { data.addEvidence(evidenceData); // Act @@ -966,7 +1159,7 @@ public void ShareUsageElement_SessionIdAndSequence_Existing() throws Exception { evidenceData.put(Constants.EVIDENCE_SESSIONID, "abcdefg-hijklmn-opqrst-uvwxyz"); evidenceData.put(Constants.EVIDENCE_SEQUENCE, 2); - try (FlowData data = pipeline.createFlowData()) { + try (FlowData data = sequencePipeline().createFlowData()) { data.addEvidence(evidenceData); // Act @@ -1260,10 +1453,10 @@ public void CheckForContinueAndResumeWhenQueueFull() throws Exception { } /** - * Default settings share headers other than Cookie, the 51Degrees cookies - * and query string parameters, and other evidence, and leave out ordinary - * cookies, other parameters, session values and the evidence that is - * never shared. + * Default settings share headers other than Cookie, the 51Degrees cookies, + * the 51Degrees query string parameters and those the Pipeline uses, and + * other evidence, and leave out ordinary cookies, other parameters, + * session values and the evidence that is never shared. */ @Test public void ShareUsageElement_DefaultSharesOnlyAcceptedEvidence() throws Exception { @@ -1307,8 +1500,9 @@ public void ShareUsageElement_DefaultSharesOnlyAcceptedEvidence() throws Excepti "some user agent", "screen width", "fiftyone parameter")) { assertTrue(output.contains(shared), shared); } + // A parameter no Flow Element accepts is not shared. for (String withheld : Arrays.asList( - "sessionid cookie", "cookie header", "password parameter", + "password parameter", "sessionid cookie", "cookie header", "email parameter", "email header", "salt value", "whole query string", "request path", "session attribute")) { assertFalse(output.contains(withheld), withheld); @@ -1353,6 +1547,79 @@ public void ShareUsageElement_RepeatWithNewSequence_NotShared() throws Exception assertEquals(1, devices.getLength(), "1 device expected"); } + /** + * Within one session, a request that differs from one already shared + * only in its sequence number is not shared again, and a later request + * with a 51D_ value changed is, with the same SessionId and its own + * Sequence. The session id and the sequence never enter the repeat + * comparison and are always written to the record. + */ + @Test + public void ShareUsageElement_OneSession_ChangedValueSharedWithItsSequence() + throws Exception { + // Arrange + tracker = null; + createShareUsage( + 1, + 3, + 1, + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>()); + String[] screenWidths = { "1920", "1920", "1024" }; + + // Act + for (int sequence = 1; sequence <= screenWidths.length; sequence++) { + Map evidenceData = new HashMap<>(); + evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); + evidenceData.put("header.user-agent", "one session user agent"); + evidenceData.put( + "cookie.51D_ScreenPixelsWidth", + screenWidths[sequence - 1]); + evidenceData.put(Constants.EVIDENCE_SESSIONID, "one session"); + evidenceData.put(Constants.EVIDENCE_SEQUENCE, sequence); + try (FlowData data = sequencePipeline().createFlowData()) { + data.addEvidence(evidenceData); + shareUsageElement.process(data); + } + } + shareUsageElement.close(); + + // Assert + Document doc = docBuilderFactory.newDocumentBuilder().parse( + new ByteArrayInputStream(connector.getBaos().toByteArray())); + NodeList devices = (NodeList) xpath.compile("//Device") + .evaluate(doc, XPathConstants.NODESET); + assertEquals(2, devices.getLength(), + "Only the first request and the one with a changed value " + + "should be shared"); + assertEquals("one session", xpath.evaluate("SessionId", devices.item(0))); + assertEquals("1", xpath.evaluate("Sequence", devices.item(0))); + assertEquals("1920", cookieValue(devices.item(0), "51D_ScreenPixelsWidth")); + assertEquals("one session", xpath.evaluate("SessionId", devices.item(1))); + assertEquals("3", xpath.evaluate("Sequence", devices.item(1))); + assertEquals("1024", cookieValue(devices.item(1), "51D_ScreenPixelsWidth")); + } + + /** + * The value a record gives for the named cookie, or null when it gives + * none. Names are compared without regard to case. + */ + private static String cookieValue(Node device, String name) { + NodeList children = device.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + Node child = children.item(i); + Node nameAttribute = child.getAttributes() == null ? + null : child.getAttributes().getNamedItem("Name"); + if ("cookie".equals(child.getNodeName()) && + nameAttribute != null && + name.equalsIgnoreCase(nameAttribute.getNodeValue())) { + return child.getTextContent(); + } + } + return null; + } + /** * The session cookie the element names is read by the tracker, whatever * the case of its name, and never sent. With sessions tracked, the same diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageInvariantValuesTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageInvariantValuesTests.java new file mode 100644 index 000000000..8887ec6d7 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageInvariantValuesTests.java @@ -0,0 +1,342 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.DataUploader; +import fiftyone.pipeline.engines.trackers.Tracker; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Pattern; + +import static fiftyone.pipeline.core.Constants.EVIDENCE_CLIENTIP_KEY; +import static fiftyone.pipeline.core.Constants.EVIDENCE_HEADER_USERAGENT_KEY; +import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The values that do not change while the element lives are written to its + * first record and again once an hour, and every record carries the + * process identifier. The sentences quoted are from + * pipeline-specification/pipeline-elements/usage-sharing-element.md in the + * specifications repository at 07e1fcedc2. + *

    + * Each record is sent in a message of its own, and the element is given a + * clock the test sets, so the hour passes without waiting for it. + */ +public class ShareUsageInvariantValuesTests { + + /** One hour, in milliseconds. */ + private static final long HOUR = 60L * 60L * 1000L; + + /** + * The elements that hold the invariant values, as lines 138 to 145 list + * them. Flow Elements are written as FlowElement and Packages as + * Package. + */ + private static final String[] INVARIANT = { + "Version", "Product", "FlowElement", "Package", "Language", + "LanguageVersion", "ServerIP", "Platform"}; + + /** + * Lines 122 to 125: "32 hexadecimal digits in groups of 8, 4, 4, 4 and + * 12 separated by hyphens". A version 4 UUID has 4 as the first digit + * of its third group and 8, 9, a or b as the first of its fourth. + */ + private static final Pattern UUID_V4 = Pattern.compile( + "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}" + + "-[0-9a-fA-F]{12}"); + + /** Keeps each message the element sends apart from the others. */ + private static class Uploader implements DataUploader { + final List messages = new ArrayList<>(); + + @Override + public OutputStream getOutputStream() { + ByteArrayOutputStream message = new ByteArrayOutputStream(); + messages.add(message); + return message; + } + + @Override + public int getResponseCode() { + return 200; + } + } + + private final List pipelines = new ArrayList<>(); + + @AfterEach + public void cleanUp() throws Exception { + for (Pipeline pipeline : pipelines) { + pipeline.close(); + } + } + + /** A Usage Sharing Element, in a Pipeline, and what it has sent. */ + private class Sharing { + final ShareUsageElement element; + final Uploader uploader = new Uploader(); + final Pipeline pipeline; + final AtomicLong now = new AtomicLong(1_000_000_000_000L); + + Sharing() throws Exception { + Tracker tracker = new Tracker() { + @Override + public boolean track(FlowData flowData) { + return true; + } + }; + element = new ShareUsageElement( + LoggerFactory.getLogger(ShareUsageElement.class), + 1, + 1, + 2, + 100, + 100, + 1, + false, + "http://51Degrees.com/test", + new ArrayList(), + new ArrayList(), + new ArrayList>(), + DEFAULT_SESSION_COOKIE_NAME, + tracker); + element.dataUploader = uploader; + element.clock = now::get; + // Package elements depend on whether the classes are loaded from + // jars, so the test gives the element one package of its own. + element.packageFinder = classes -> { + TreeMap packages = new TreeMap<>(); + packages.put("com.51degrees:pipeline.engines.fiftyone", "4.5.0"); + return packages; + }; + pipeline = new PipelineBuilder() + .addFlowElement(new SequenceElement( + LoggerFactory.getLogger(SequenceElement.class))) + .addFlowElement(element) + .build(); + pipelines.add(pipeline); + } + + /** Send one record and return the Device element written for it. */ + Element send() throws Exception { + int before = uploader.messages.size(); + Map evidence = new HashMap<>(); + evidence.put(EVIDENCE_HEADER_USERAGENT_KEY, "Mozilla/5.0"); + evidence.put(EVIDENCE_CLIENTIP_KEY, "192.0.2.1"); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidence); + data.process(); + assertNotNull(element.getSendDataFuture()); + element.getSendDataFuture().get(); + } + assertEquals(before + 1, uploader.messages.size(), + "One message for each record"); + Document doc = DocumentBuilderFactory.newInstance() + .newDocumentBuilder() + .parse(new ByteArrayInputStream( + uploader.messages.get(before).toByteArray())); + NodeList devices = doc.getElementsByTagName("Device"); + assertEquals(1, devices.getLength(), "One record in the message"); + return (Element) devices.item(0); + } + + /** Move the clock the element reads on by the time given. */ + void advance(long millis) { + now.addAndGet(millis); + } + } + + private static int count(Element device, String name) { + int count = 0; + for (Node child = device.getFirstChild(); + child != null; + child = child.getNextSibling()) { + if (child.getNodeType() == Node.ELEMENT_NODE && + child.getNodeName().equals(name)) { + count++; + } + } + return count; + } + + private static void assertCarriesInvariantValues(Element device) { + for (String name : INVARIANT) { + assertTrue(count(device, name) > 0, + name + " is an invariant value and the record should carry it"); + } + } + + private static void assertLeavesOutInvariantValues(Element device) { + // Lines 207 to 210: "A record that leaves them out has no Version, + // Product, FlowElement, Package, Language, LanguageVersion, ServerIP + // or Platform element, and is otherwise the same." + for (String name : INVARIANT) { + assertEquals(0, count(device, name), + name + " is an invariant value and the record should leave it out"); + } + for (String name : new String[] { + "ProcessId", "SessionId", "Sequence", "DateSent", "ClientIP"}) { + assertEquals(1, count(device, name), + name + " is written to every record"); + } + assertEquals(1, count(device, "header"), "The Evidence is written"); + } + + private static String processId(Element device) { + assertEquals(1, count(device, "ProcessId"), + "Every record carries one ProcessId"); + return device.getElementsByTagName("ProcessId").item(0) + .getTextContent(); + } + + /** + * Lines 150 and 151: "The invariant values MUST be written to the first + * record the element writes." Line 214 puts ProcessId first in the + * record. + */ + @Test + public void FirstRecord_CarriesTheInvariantValues() throws Exception { + Sharing sharing = new Sharing(); + Element first = sharing.send(); + + assertCarriesInvariantValues(first); + Node firstChild = first.getFirstChild(); + while (firstChild.getNodeType() != Node.ELEMENT_NODE) { + firstChild = firstChild.getNextSibling(); + } + assertEquals("ProcessId", firstChild.getNodeName(), + "ProcessId comes first, as in the specification's example"); + } + + /** + * Lines 152 and 153: "They MUST NOT be written to any other record." A + * record written a moment less than an hour after the first leaves + * them out. + */ + @Test + public void RecordsAfterTheFirst_LeaveThemOut_UntilTheHourIsUp() + throws Exception { + Sharing sharing = new Sharing(); + assertCarriesInvariantValues(sharing.send()); + + sharing.advance(1); + assertLeavesOutInvariantValues(sharing.send()); + sharing.advance(HOUR / 2); + assertLeavesOutInvariantValues(sharing.send()); + sharing.advance(HOUR / 2 - 2); + assertLeavesOutInvariantValues(sharing.send()); + } + + /** + * Lines 151 and 152: "They MUST be written again to the first record the + * element writes once an hour or more has passed since it last wrote + * them." Lines 159 and 160: "The hour is measured from when the values + * were last written to a record". So the hour starts again from the + * record that carried them again, not from the first record. + */ + @Test + public void HourUp_WritesThemAgain_AndTheNextHourStartsThen() + throws Exception { + Sharing sharing = new Sharing(); + assertCarriesInvariantValues(sharing.send()); + + // No record is written for an hour and a half. + sharing.advance(HOUR + HOUR / 2); + assertCarriesInvariantValues(sharing.send()); + // An hour after the first record, but not after the second. + sharing.advance(HOUR / 2); + assertLeavesOutInvariantValues(sharing.send()); + // Exactly an hour after the record that last carried them. + sharing.advance(HOUR / 2); + assertCarriesInvariantValues(sharing.send()); + assertLeavesOutInvariantValues(sharing.send()); + } + + /** + * Lines 115 and 116: "The element MUST create a process identifier when + * it is built, and MUST write it to every record as the `ProcessId` + * element." Lines 238 and 239: "the same in every record the element + * writes." + */ + @Test + public void OneElement_KeepsOneProcessId_OnEveryRecord() throws Exception { + Sharing sharing = new Sharing(); + String first = processId(sharing.send()); + + assertTrue(UUID_V4.matcher(first).matches(), + first + " is not a version 4 UUID in the string format"); + sharing.advance(1); + assertEquals(first, processId(sharing.send())); + sharing.advance(HOUR); + assertEquals(first, processId(sharing.send()), + "The record that carries the invariant values again keeps it"); + sharing.advance(1); + assertEquals(first, processId(sharing.send())); + } + + /** + * Lines 128 to 131: "The identifier is never stored, so a process that + * starts again creates a new one. A process that builds more than one + * Usage Sharing Element, for example one for each of two Pipelines, + * creates an identifier for each". A process that starts again builds + * its element afresh, so a fresh element stands for it here. + */ + @Test + public void FreshElement_CreatesANewProcessId() throws Exception { + Sharing before = new Sharing(); + String first = processId(before.send()); + + Sharing after = new Sharing(); + Element record = after.send(); + String second = processId(record); + + assertTrue(UUID_V4.matcher(second).matches(), + second + " is not a version 4 UUID in the string format"); + assertNotEquals(first.toLowerCase(), second.toLowerCase(), + "A fresh element creates a new identifier"); + assertCarriesInvariantValues(record); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java index 65a920482..afa9d2bba 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java @@ -46,6 +46,7 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @@ -126,6 +127,7 @@ private static Map evidence() { evidence.put("header.x-blocked", "blocked"); evidence.put("header.cookie", "a=b"); evidence.put("cookie.plain", "cookie"); + evidence.put("cookie.51D_plain", "51d-cookie"); evidence.put("session.plain", "session"); evidence.put("query.plain", "query"); evidence.put("query.id.email", "a@example.com"); @@ -177,7 +179,9 @@ public void defaults_ShareOnlyWhatTheRulesAllow() throws Exception { assertTrue(shared(sent, "header", "user-agent")); assertFalse(shared(sent, "header", "x-blocked")); assertFalse(shared(sent, "cookie", "plain")); + assertTrue(shared(sent, "cookie", "51D_plain")); assertFalse(shared(sent, "session", "plain")); + // No Flow Element in the Pipeline accepts the parameter. assertFalse(shared(sent, "query", "plain")); } @@ -188,9 +192,12 @@ public void shareAllEvidence_FromConfiguration_SharesEverythingButNeverShared() assertTrue(shared(sent, "header", "user-agent")); assertTrue(shared(sent, "header", "x-blocked")); - assertTrue(shared(sent, "cookie", "plain")); + assertFalse(shared(sent, "cookie", "plain")); + assertTrue(shared(sent, "cookie", "51D_plain")); assertTrue(shared(sent, "session", "plain")); - assertTrue(shared(sent, "query", "plain")); + // Not even all Evidence shares a parameter the Pipeline does not + // use. + assertFalse(shared(sent, "query", "plain")); assertFalse(shared(sent, "query", "id.email")); assertFalse(shared(sent, "id", "email")); assertFalse(shared(sent, "id", "salt")); @@ -209,14 +216,18 @@ public void shareAllEvidence_False_FromConfiguration_KeepsTheRules() assertFalse(shared(sent, "query", "plain")); } + /** + * The deprecated share all query string parameters option has no + * effect, so a parameter the Pipeline does not use is not shared. + */ @Test - public void shareAllQueryStringParameters_FromConfiguration_SharesEveryParameter() + public void shareAllQueryStringParameters_FromConfiguration_HasNoEffect() throws Exception { Document sent = send( options("ShareAllQueryStringParameters", "true"), evidence()); - assertTrue(shared(sent, "query", "plain")); + assertFalse(shared(sent, "query", "plain")); assertFalse(shared(sent, "query", "id.email")); assertFalse(shared(sent, "server", "request-query")); // The other rules still apply. @@ -236,13 +247,17 @@ public void shareAllQueryStringParameters_False_FromConfiguration_KeepsTheRules( assertFalse(shared(sent, "query", "plain")); } + /** + * Share all Evidence is off by default and share all query string + * parameters is on. + */ @Test - public void builder_ShareOptionsDefaultToFalse() { + public void builder_ShareOptionDefaults() { ShareUsageBuilder builder = new ShareUsageBuilder( LoggerFactory.getILoggerFactory()); assertFalse(builder.isShareAllEvidence()); - assertFalse(builder.isShareAllQueryStringParameters()); + assertTrue(builder.isShareAllQueryStringParameters()); } @Test @@ -251,8 +266,14 @@ public void keysAreReadWithoutRegardToCase() throws Exception { evidence.put("Server.Client-IP", "1.2.3.4"); evidence.put("Query.Session-ID", "session-1"); evidence.put("Query.Sequence", "5"); + // The Sequence Element accepts the session id and sequence, so the + // Pipeline uses them and they are shared. + PipelineOptions options = options(); + ElementOptions sequence = new ElementOptions(); + sequence.builderName = "SequenceElement"; + options.elements.add(0, sequence); - Document sent = send(options(), evidence); + Document sent = send(options, evidence); assertEquals("1.2.3.4", sent.getElementsByTagName("ClientIP").item(0).getTextContent()); @@ -262,6 +283,41 @@ public void keysAreReadWithoutRegardToCase() throws Exception { sent.getElementsByTagName("Sequence").item(0).getTextContent()); } + /** + * The never-shared keys are matched without regard to case under any + * default locale. Under a Turkish default locale "I" lower-cases to a + * dotless "i", so a comparison in the default locale would let + * "ID.EMAIL" through. + */ + @Test + public void neverSharedKeys_TurkishLocale_StillRefused() { + Locale original = Locale.getDefault(); + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + try { + EvidenceKeyFilterShareUsage filter = new EvidenceKeyFilterShareUsage( + new ArrayList(), + new ArrayList(), + false, + DEFAULT_SESSION_COOKIE_NAME); + EvidenceKeyFilterShareUsage shareAll = + new EvidenceKeyFilterShareUsage( + new ArrayList(), + new ArrayList(), + false, + DEFAULT_SESSION_COOKIE_NAME, + true, + true); + for (String key : new String[] { + "ID.EMAIL", "Query.ID.Email", "HEADER.COOKIE", + "Server.Request-Path", "SERVER.REQUEST-QUERY", "ID.SALT"}) { + assertFalse(filter.include(key), key); + assertFalse(shareAll.include(key), key); + } + } finally { + Locale.setDefault(original); + } + } + @Test public void tracker_BuildsOneCache() { CountingCacheBuilder builder = new CountingCacheBuilder(); diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageParameterTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageParameterTests.java new file mode 100644 index 000000000..88c9d60bb --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageParameterTests.java @@ -0,0 +1,244 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.FlowElementBase; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.DataUploader; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * A query string or form parameter is shared only when its name starts with + * 51D_ or a Flow Element in the Pipeline accepts it, for GET and POST + * requests alike, whatever the configuration. The deprecated included query + * string parameters and share all query string parameters options, and + * sharing all Evidence, share no other parameter. + * @see Specification + */ +public class ShareUsageParameterTests { + + private static final String WEB_REQUEST = "server.http-request"; + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + private static class Capture implements DataUploader { + private final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + @Override + public OutputStream getOutputStream() { + return bytes; + } + + @Override + public int getResponseCode() { + return 200; + } + + String text() { + return new String(bytes.toByteArray(), StandardCharsets.UTF_8); + } + } + + /** + * Accepts query.user-agent and the form parameter query.used-form-field, + * so the Pipeline uses them. + */ + private static class Acceptor + extends FlowElementBase { + Acceptor() { + super(LoggerFactory.getLogger(Acceptor.class), + (flowData, element) -> null); + } + + @Override + protected void processInternal(FlowData data) { + } + + @Override + public String getElementDataKey() { + return "acceptor"; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist( + Arrays.asList("query.user-agent", "query.used-form-field")); + } + + @Override + public List getProperties() { + return Collections.emptyList(); + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + } + + /** + * Share one request and give the record that was sent. + * @param options which of the options that must not widen what is + * shared to set: none, included, shareall or everything + */ + private String share(String options) throws Exception { + Capture capture = new Capture(); + ShareUsageBuilder builder = new ShareUsageBuilder( + loggerFactory, + capture); + builder.setMinimumEntriesPerMessage(1); + if (options.equals("included") || options.equals("everything")) { + builder.setIncludedQueryStringParameters("password,formpassword"); + } + if (options.equals("shareall") || options.equals("everything")) { + builder.setShareAllQueryStringParameters(true); + } + if (options.equals("everything")) { + builder.setShareAllEvidence(true); + } + ShareUsageElement shareUsage = builder.build(); + + Map post = new HashMap<>(); + post.put("query.formpassword", "form-secret-value"); + post.put("query.51D_FormValue", "form-51d-value"); + post.put("query.used-form-field", "form-accepted-value"); + post.put("header.user-agent", "header-value"); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(new Acceptor()) + .addFlowElement(shareUsage) + .setAutoCloseElements(true) + .build()) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.password", "query-secret-value"); + data.addEvidence("query.51D_QueryValue", "query-51d-value"); + data.addEvidence("query.user-agent", "accepted-value"); + // The web request view of a POST, which carries the form + // parameters as query. keys. + data.addEvidence(WEB_REQUEST, post); + data.process(); + } + } + return capture.text(); + } + + /** + * Share two requests that differ only in one query parameter and give + * how many of them were sent. The repeat Evidence check hashes the + * values that would be shared, so the second is a repeat only when the + * parameter is not shared. + * @param key the query parameter whose value differs + */ + private int sharedOfTwo(String key) throws Exception { + Capture capture = new Capture(); + ShareUsageBuilder builder = new ShareUsageBuilder( + loggerFactory, + capture); + builder.setMinimumEntriesPerMessage(1); + ShareUsageElement shareUsage = builder.build(); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(new Acceptor()) + .addFlowElement(shareUsage) + .setAutoCloseElements(true) + .build()) { + for (int i = 1; i <= 2; i++) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "same-header"); + data.addEvidence(key, "value-" + i); + data.process(); + } + } + } + String record = capture.text(); + return record.split("same-header", -1).length - 1; + } + + @ParameterizedTest + @ValueSource(strings = {"query.user-agent", "query.51D_Value"}) + public void repeats_DifferInASharedParameter_BothShared(String key) + throws Exception { + assertEquals(2, sharedOfTwo(key)); + } + + @ParameterizedTest + @ValueSource(strings = {"query.password"}) + public void repeats_DifferInAParameterNotShared_OneShared(String key) + throws Exception { + assertEquals(1, sharedOfTwo(key)); + } + + @ParameterizedTest + @ValueSource(strings = {"none", "included", "shareall", "everything"}) + public void parameters_SharedOnlyWhen51DOrUsed(String options) + throws Exception { + String record = share(options); + + assertTrue(record.contains("header-value"), record); + assertTrue(record.contains("query-51d-value"), record); + assertTrue(record.contains("form-51d-value"), record); + assertTrue(record.contains("accepted-value"), record); + assertFalse(record.contains("query-secret-value"), record); + assertFalse(record.contains("form-secret-value"), record); + } + + /** + * A form parameter sent only in the body of a POST, which a Flow Element + * in the Pipeline accepts, is shared under every setup, just as a used + * query string parameter in the Evidence is. + */ + @ParameterizedTest + @ValueSource(strings = {"none", "included", "shareall", "everything"}) + public void postParameter_UsedByThePipeline_Shared(String options) + throws Exception { + String record = share(options); + + assertTrue(record.contains("form-accepted-value"), record); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsagePositionTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsagePositionTests.java new file mode 100644 index 000000000..ecae95ca9 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsagePositionTests.java @@ -0,0 +1,180 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.configuration.ElementOptions; +import fiftyone.pipeline.core.configuration.PipelineOptions; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.DataUploader; +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngine; +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngineBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * The usage sharing element is the last Flow Element of every Pipeline a + * builder makes, and a Pipeline that puts it anywhere else, or in a parallel + * group, is refused when it is built, in code or from a configuration. + * @see Specification + */ +@SuppressWarnings("rawtypes") +public class ShareUsagePositionTests { + + private static final String MUST_BE_LAST = + "must be the last Flow Element and outside any parallel group"; + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + // Sends nothing anywhere when the element is closed. + private ShareUsageElement shareUsage() throws Exception { + return new ShareUsageBuilder(loggerFactory, mock(DataUploader.class)) + .build(); + } + + private EmptyEngine engine() throws Exception { + return new EmptyEngineBuilder(loggerFactory).build(); + } + + private static void assertRefused(Executable build) { + PipelineConfigurationException e = assertThrows( + PipelineConfigurationException.class, + build::run, + "A Pipeline with usage sharing anywhere but last was built."); + assertTrue(e.getMessage().startsWith("The usage sharing element " + + MUST_BE_LAST), e.getMessage()); + } + + private interface Executable { + void run() throws Exception; + } + + private static void assertLast(Pipeline pipeline) { + List elements = pipeline.getFlowElements(); + assertTrue( + elements.get(elements.size() - 1) instanceof ShareUsageElement, + "Usage sharing is not the last Flow Element."); + } + + @Test + public void ShareUsage_BeforeAnotherElement_Refused() throws Exception { + ShareUsageElement shareUsage = shareUsage(); + EmptyEngine engine = engine(); + try { + assertRefused(() -> new PipelineBuilder(loggerFactory) + .addFlowElement(shareUsage) + .addFlowElement(engine) + .build() + .close()); + } finally { + shareUsage.close(); + engine.close(); + } + } + + @Test + public void ShareUsage_InAParallelGroup_Refused() throws Exception { + ShareUsageElement shareUsage = shareUsage(); + EmptyEngine engine = engine(); + try { + assertRefused(() -> new PipelineBuilder(loggerFactory) + .addFlowElementsParallel( + new FlowElement[]{engine, shareUsage}) + .build() + .close()); + } finally { + shareUsage.close(); + engine.close(); + } + } + + @Test + public void ShareUsage_Last_Built() throws Exception { + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine()) + .addFlowElement(shareUsage()) + .setAutoCloseElements(true) + .build()) { + assertLast(pipeline); + } + } + + private static ElementOptions element(String builderName) { + ElementOptions element = new ElementOptions(); + element.builderName = builderName; + return element; + } + + @Test + public void ShareUsage_FirstInAConfiguration_Refused() { + PipelineOptions options = new PipelineOptions(); + options.elements.add(element("ShareUsage")); + options.elements.add(element("SetHeadersElement")); + assertRefused(() -> new PipelineBuilder(loggerFactory) + .buildFromConfiguration(options) + .close()); + } + + @Test + public void ShareUsage_InAParallelGroupInAConfiguration_Refused() { + ElementOptions group = new ElementOptions(); + group.subElements.add(element("SetHeadersElement")); + group.subElements.add(element("ShareUsage")); + PipelineOptions options = new PipelineOptions(); + options.elements.add(group); + assertRefused(() -> new PipelineBuilder(loggerFactory) + .buildFromConfiguration(options) + .close()); + } + + @Test + public void FiftyOnePipelineBuilder_PutsShareUsageLast() throws Exception { + try (Pipeline pipeline = new FiftyOnePipelineBuilder(loggerFactory) + .addFlowElement(engine()) + .setAutoCloseElements(true) + .build()) { + assertLast(pipeline); + } + } + + @Test + public void FiftyOnePipelineBuilder_FromAConfiguration_PutsShareUsageLast() + throws Exception { + PipelineOptions options = new PipelineOptions(); + options.elements.add(element("SetHeadersElement")); + try (Pipeline pipeline = new FiftyOnePipelineBuilder(loggerFactory) + .buildFromConfiguration(options)) { + assertLast(pipeline); + } + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageRecordFormTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageRecordFormTests.java new file mode 100644 index 000000000..315509987 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageRecordFormTests.java @@ -0,0 +1,398 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.DataUploader; +import fiftyone.pipeline.engines.services.HttpClient; +import fiftyone.pipeline.engines.trackers.Tracker; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.NodeList; + +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static fiftyone.pipeline.core.Constants.EVIDENCE_CLIENTIP_KEY; +import static fiftyone.pipeline.core.Constants.EVIDENCE_HEADER_USERAGENT_KEY; +import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The usage sharing record is written as the specification's Processing + * section sets it out. Each test sends one record through a Pipeline that + * holds a Sequence Element and the Usage Sharing Element, and reads what + * the element wrote. + */ +public class ShareUsageRecordFormTests { + + /** Keeps what the element writes. */ + private static class Uploader implements DataUploader { + final ByteArrayOutputStream written = new ByteArrayOutputStream(); + + @Override + public OutputStream getOutputStream() { + return written; + } + + @Override + public int getResponseCode() { + return 200; + } + } + + /** Keeps the headers the element sends through an HTTP client. */ + private static class HeaderClient implements HttpClient { + Map sent; + + @Override + public HttpURLConnection connect(URL url) { + return new HttpURLConnection(url) { + @Override + public int getResponseCode() { + return 200; + } + + @Override + public String getResponseMessage() { + return ""; + } + + @Override + public void disconnect() { + } + + @Override + public boolean usingProxy() { + return false; + } + + @Override + public void connect() { + } + }; + } + + @Override + public String postData( + HttpURLConnection connection, + Map headers, + byte[] data) { + sent = new HashMap<>(headers); + return ""; + } + + @Override + public String getResponseString(HttpURLConnection connection) { + return ""; + } + + @Override + public String getResponseString( + HttpURLConnection connection, + Map headers) { + return ""; + } + } + + private static final String DATE_PATTERN = "yyyy-MM-dd'T'HH:mm:ss"; + + private Pipeline pipeline; + private ShareUsageElement element; + private Uploader uploader; + + @AfterEach + public void cleanUp() throws Exception { + if (pipeline != null) { + pipeline.close(); + } + } + + /** + * Build a Pipeline of a Sequence Element and a Usage Sharing Element + * that sends every record as soon as it has it. + */ + private void build() throws Exception { + SequenceElement sequence = new SequenceElement( + LoggerFactory.getLogger(SequenceElement.class)); + Tracker tracker = new Tracker() { + @Override + public boolean track(FlowData flowData) { + return true; + } + }; + element = new ShareUsageElement( + LoggerFactory.getLogger(ShareUsageElement.class), + 1, + 1, + 2, + 100, + 100, + 1, + true, + "http://51Degrees.com/test", + new ArrayList(), + new ArrayList(), + new ArrayList>(), + DEFAULT_SESSION_COOKIE_NAME, + tracker); + uploader = new Uploader(); + element.dataUploader = uploader; + // The Package elements depend on whether the build loads the + // classes from jars, so these tests find none and + // ShareUsageElementTests checks them. + element.packageFinder = classes -> new TreeMap<>(); + pipeline = new PipelineBuilder() + .addFlowElement(sequence) + .addFlowElement(element) + .build(); + } + + /** Send one record and return the text the element wrote. */ + private String send() throws Exception { + Map evidence = new HashMap<>(); + evidence.put(EVIDENCE_HEADER_USERAGENT_KEY, " & co"); + evidence.put(EVIDENCE_CLIENTIP_KEY, "192.0.2.1"); + return send(evidence); + } + + /** + * Send one record of the Evidence given and return the text the + * element wrote. + */ + private String send(Map evidence) throws Exception { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidence); + data.process(); + assertNotNull(element.getSendDataFuture()); + element.getSendDataFuture().get(); + } + return new String( + uploader.written.toByteArray(), StandardCharsets.UTF_8); + } + + private static Document parse(String xml) throws Exception { + return DocumentBuilderFactory.newInstance().newDocumentBuilder() + .parse(new ByteArrayInputStream( + xml.getBytes(StandardCharsets.UTF_8))); + } + + private static String text(Document doc, String name) { + NodeList nodes = doc.getElementsByTagName(name); + assertEquals(1, nodes.getLength(), "One " + name + " expected"); + return nodes.item(0).getTextContent(); + } + + /** The root is Devices alone, with no attribute. */ + @Test + public void Root_IsDevicesWithNoAttribute() throws Exception { + build(); + String xml = send(); + + assertTrue(xml.startsWith(""), xml); + assertTrue(xml.endsWith(""), xml); + assertEquals(0, + parse(xml).getDocumentElement().getAttributes().getLength()); + } + + /** + * DateSent is the time in UTC, whatever time zone the JVM runs in. + * {@code DATE_FMT} keeps the zone that was the default when the class + * loaded, so the test gives it the zone a JVM started in that zone + * has. + */ + @Test + @SuppressWarnings("deprecation") + public void DateSent_IsUtc_WhenTheDefaultTimeZoneIsNot() throws Exception { + TimeZone defaultBefore = TimeZone.getDefault(); + TimeZone formatBefore = ShareUsageBase.DATE_FMT.getTimeZone(); + try { + TimeZone.setDefault(TimeZone.getTimeZone("Pacific/Kiritimati")); + ShareUsageBase.DATE_FMT.setTimeZone(TimeZone.getDefault()); + build(); + + long before = System.currentTimeMillis(); + String sent = text(parse(send()), "DateSent"); + long after = System.currentTimeMillis(); + + SimpleDateFormat utc = new SimpleDateFormat(DATE_PATTERN); + utc.setTimeZone(TimeZone.getTimeZone("UTC")); + long written = utc.parse(sent).getTime(); + // DateSent has no part of a second, so allow one either side. + assertTrue( + written >= before - 1000 && written <= after + 1000, + "DateSent was " + sent + ", which is not the time in UTC, " + + utc.format(before)); + } finally { + TimeZone.setDefault(defaultBefore); + ShareUsageBase.DATE_FMT.setTimeZone(formatBefore); + } + } + + /** Each FlowElement holds the fully qualified name of the class. */ + @Test + public void FlowElement_IsTheFullyQualifiedClassName() throws Exception { + build(); + NodeList names = parse(send()).getElementsByTagName("FlowElement"); + + assertEquals(2, names.getLength()); + assertEquals( + "fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement", + names.item(0).getTextContent()); + assertEquals( + "fiftyone.pipeline.engines.fiftyone.flowelements." + + "ShareUsageElement", + names.item(1).getTextContent()); + } + + /** Platform holds the name of the operating system and its version. */ + @Test + public void Platform_IsTheNameAndTheVersion() throws Exception { + String name = System.getProperty("os.name"); + String version = System.getProperty("os.version"); + assertNotNull(version, "This JVM gives no os.version to test with"); + build(); + + assertEquals( + name + " " + version, text(parse(send()), "Platform")); + } + + /** + * ClientIP is written, empty, when the Evidence holds no client IP, as + * the specification's example does not mark it optional. It stays in + * its place between LanguageVersion and ServerIP. + */ + @Test + public void ClientIP_IsWrittenEmpty_WhenTheEvidenceHasNone() + throws Exception { + build(); + Map evidence = new HashMap<>(); + evidence.put(EVIDENCE_HEADER_USERAGENT_KEY, " & co"); + String xml = send(evidence); + + assertEquals("", text(parse(xml), "ClientIP")); + assertTrue( + Pattern.compile( + "| ?/>)") + .matcher(xml).find(), + xml); + } + + /** The message is sent as text/xml, compressed with gzip. */ + @Test + public void ContentType_IsTextXml() throws Exception { + build(); + + assertEquals("text/xml", element.headers.get("Content-Type")); + assertEquals("gzip", element.headers.get("Content-Encoding")); + + // The same headers go to an HTTP client the element is given. + HeaderClient client = new HeaderClient(); + element.httpClient = client; + send(); + assertEquals("text/xml", client.sent.get("Content-Type")); + assertEquals("gzip", client.sent.get("Content-Encoding")); + } + + /** + * The whole record, compared as text. It is the element's first + * record, so it carries the invariant values. ProcessId, DateSent and + * ServerIP are read from the record, as they differ from run to run, + * and the values that + * name the runtime are the ones this JVM gives. Version is written + * even when the classes carry no version, as when the tests run from + * the build's class folders, and is then empty. + */ + @Test + public void WholeRecord_MatchesTheSpecificationExample() throws Exception { + build(); + String xml = send(); + + Matcher date = Pattern.compile( + "(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2})") + .matcher(xml); + assertTrue(date.find(), xml); + Matcher process = Pattern.compile( + "([^<]+)").matcher(xml); + assertTrue(process.find(), xml); + Matcher session = Pattern.compile( + "([^<]+)").matcher(xml); + assertTrue(session.find(), xml); + Matcher server = Pattern.compile( + "([^<]*)").matcher(xml); + assertTrue(server.find(), xml); + String version = Pipeline.class.getPackage() + .getImplementationVersion(); + + StringBuilder expected = new StringBuilder(); + expected.append("") + .append("").append(process.group(1)) + .append("") + .append("").append(session.group(1)) + .append("") + .append("1") + .append("").append(date.group(1)).append(""); + expected.append("").append(version == null ? "" : version) + .append(""); + expected.append("Pipeline") + .append("fiftyone.pipeline.engines.fiftyone." + + "flowelements.SequenceElement") + .append("fiftyone.pipeline.engines.fiftyone." + + "flowelements.ShareUsageElement") + .append("java") + .append("") + .append(System.getProperty("java.version")) + .append("") + .append("192.0.2.1") + .append("").append(server.group(1)) + .append("") + .append("").append(System.getProperty("os.name")) + .append(" ").append(System.getProperty("os.version")) + .append("") + .append("

    ") + .append("<iPhone> & co
    ") + .append(""); + + assertEquals(expected.toString(), xml); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/WritesAfterProcessingTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/WritesAfterProcessingTests.java new file mode 100644 index 000000000..07d21e169 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/WritesAfterProcessingTests.java @@ -0,0 +1,123 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.fiftyone.data.SequenceData; +import fiftyone.pipeline.engines.fiftyone.data.SetHeadersData; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.HashMap; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * The check, for the Flow Elements in this module, that processes each one, + * then writes to its Element Data and expects the error: the + * {@link SequenceElement}, the {@link SetHeadersElement}, and the + * {@link ShareUsageElement}, which adds no Element Data, so has none to + * write to. + * @see Specification + */ +public class WritesAfterProcessingTests { + + private static final ILoggerFactory LOGGER_FACTORY = + LoggerFactory.getILoggerFactory(); + + static void assertWritesFail(ElementData data) { + assertThrows( + IllegalStateException.class, + () -> data.put("after", "value"), + "put"); + assertThrows( + IllegalStateException.class, + () -> data.populateFromMap( + Collections.singletonMap("after", "value")), + "populateFromMap"); + } + + @Test + public void sequenceAndSetHeaders_ElementDataRefusesWrites() + throws Exception { + SequenceElement sequence = + new SequenceElementBuilder(LOGGER_FACTORY).build(); + SetHeadersElement setHeaders = + new SetHeadersElementBuilder(LOGGER_FACTORY).build(); + try (Pipeline pipeline = new PipelineBuilder(LOGGER_FACTORY) + .addFlowElement(sequence) + .addFlowElement(setHeaders) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + data.process(); + assertEquals(0, data.getErrors().size(), + String.valueOf(data.getErrors())); + + SequenceData sequenceData = + (SequenceData) data.getFromElement(sequence); + assertWritesFail(sequenceData); + assertThrows( + IllegalStateException.class, + () -> sequenceData.setSessionId("after"), + "setSessionId"); + assertThrows( + IllegalStateException.class, + () -> sequenceData.setSequence(2), + "setSequence"); + + SetHeadersData headersData = data.getFromElement(setHeaders); + assertWritesFail(headersData); + assertThrows( + IllegalStateException.class, + () -> headersData.setResponseHeaderDictionary( + new HashMap()), + "setResponseHeaderDictionary"); + } + } + + @Test + public void shareUsage_AddsNoElementData() throws Exception { + ShareUsageBuilder builder = new ShareUsageBuilder(LOGGER_FACTORY); + builder.setSharePercentage(0); + ShareUsageElement shareUsage = builder.build(); + try (Pipeline pipeline = new PipelineBuilder(LOGGER_FACTORY) + .addFlowElement(shareUsage) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "test"); + data.process(); + + assertFalse(data.getDataKeys().contains( + shareUsage.getElementDataKey())); + } + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/performance/ShareUsageOverheadTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/performance/ShareUsageOverheadTests.java index 9583850a7..71a362e7e 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/performance/ShareUsageOverheadTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/performance/ShareUsageOverheadTests.java @@ -40,7 +40,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.ILoggerFactory; -import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; @@ -50,6 +49,7 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -58,8 +58,13 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +/** + * Times Process calls through the usage sharing element. Every call in a + * test carries the same Evidence, so after the first call the tracker turns + * each one away as a repeat and nothing more is queued. The measured rounds + * time that path, which finds the values to share and checks the tracker. + */ public class ShareUsageOverheadTests { - static Logger logger = LoggerFactory.getLogger("testLogger"); private Pipeline pipeline; private static final Map overheadMillis = new LinkedHashMap<>(); @@ -115,50 +120,37 @@ public void cleanUp() throws Exception { pipeline.close(); } - // The FlowData are all processed before any is closed, so that closing - // them is not part of the time measured. They are closed together in a - // finally block rather than one by one with try-with-resources. - @Test - @SuppressWarnings("PMD.FlowDataInTryWithResources") - public void ShareUsageOverhead_SingleEvidence() throws Exception { - int iterations = 10_000; - double maxOverheadPerCall = 0.1; - List data = new ArrayList<>(); - long start; - long end; - try { - for (int i = 0; i < iterations; i++) { - FlowData flowData = pipeline.createFlowData(); - data.add(flowData); - flowData.addEvidence("header.user-agent", "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405"); - } + private static final String USER_AGENT = + "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) " + + "AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405"; - start = System.currentTimeMillis(); - for (FlowData entry : data) { - entry.process(); - } - end = System.currentTimeMillis(); - } finally { - closeAll(data); - } + /** + * Rounds measured, and the number discarded first so that the JIT has + * compiled the code. The median of the rounds is checked, so one round + * slowed by a busy runner or a garbage collection does not fail a test, + * whilst a cost on every call does. + */ + private static final int MeasuredRounds = 7; + private static final int WarmUpRounds = 2; - double msOverheadPerCall = (double)(end - start) / iterations; - overheadMillis.put("ShareUsageOverhead_SingleEvidence_ms", msOverheadPerCall); - System.out.println("ShareUsageOverhead_SingleEvidence: " + msOverheadPerCall + "ms per call"); - logger.info("Overhead was {} millis", msOverheadPerCall); - Assertions.assertTrue(msOverheadPerCall < maxOverheadPerCall, - "Pipeline with share usage overhead per Process call was " + - msOverheadPerCall + "ms. Maximum permitted is " + maxOverheadPerCall); - } + /** + * The most the median Process call may take. Each limit is about ten + * times the highest median the CI runners measure at their usual speed, + * so a runner several times slower than usual still passes. + */ + private static final double SingleEvidenceLimitMillis = 0.07; + private static final double ThousandEvidenceLimitMillis = 40; - // See ShareUsageOverhead_SingleEvidence for why the FlowData are closed - // in a finally block. - @Test + /** + * Returns the milliseconds per Process call, for one round of calls with + * the evidence given. The FlowData are all processed before any is closed, + * so that closing them is not part of the time measured. They are closed + * together in a finally block rather than one by one with + * try-with-resources. + */ @SuppressWarnings("PMD.FlowDataInTryWithResources") - public void ShareUsageOverhead_ThousandEvidence() throws Exception { - int iterations = 1_000; - int evidenceCount = 1_000; - int maxOverheadPerCallMillis = 10; + private double measureRound(int iterations, int evidenceCount) + throws Exception { List data = new ArrayList<>(); long start; long end; @@ -169,24 +161,68 @@ public void ShareUsageOverhead_ThousandEvidence() throws Exception { for (int j = 0; j < evidenceCount; j++) { flowData.addEvidence("header." + j, j); } - flowData.addEvidence("header.user-agent", "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405"); + flowData.addEvidence("header.user-agent", USER_AGENT); } - start = System.currentTimeMillis(); + start = System.nanoTime(); for (FlowData entry : data) { entry.process(); } - end = System.currentTimeMillis(); + end = System.nanoTime(); } finally { closeAll(data); } + return (end - start) / 1_000_000.0 / iterations; + } + + /** + * Returns the milliseconds per Process call of each measured round, + * lowest first, after the warm up rounds. + */ + private double[] measureRounds(int iterations, int evidenceCount) + throws Exception { + for (int i = 0; i < WarmUpRounds; i++) { + measureRound(iterations, evidenceCount); + } + double[] rounds = new double[MeasuredRounds]; + for (int i = 0; i < rounds.length; i++) { + rounds[i] = measureRound(iterations, evidenceCount); + } + Arrays.sort(rounds); + return rounds; + } + + /** + * Fails when the median round is not below the limit. The message lists + * every round, because the build log shows only the message, and the + * rounds tell a runner slow for the whole test from a slow round or two. + */ + private static void assertBelowLimit(double[] rounds, double limitMillis) { + double median = rounds[rounds.length / 2]; + Assertions.assertTrue(median < limitMillis, + "Pipeline with share usage overhead per Process call was " + + median + "ms. Maximum permitted is " + limitMillis + + ". Rounds in ms per call " + Arrays.toString(rounds)); + } + + @Test + public void ShareUsageOverhead_RepeatedSingleEvidence() throws Exception { + int iterations = 10_000; + + double[] rounds = measureRounds(iterations, 0); + overheadMillis.put("ShareUsageOverhead_SingleEvidence_ms", + rounds[rounds.length / 2]); + assertBelowLimit(rounds, SingleEvidenceLimitMillis); + } + + @Test + public void ShareUsageOverhead_RepeatedThousandEvidence() throws Exception { + int iterations = 250; + int evidenceCount = 1_000; - double msOverheadPerCall = (double)(end - start) / iterations; - overheadMillis.put("ShareUsageOverhead_HundredEvidence_ms", msOverheadPerCall); - System.out.println("ShareUsageOverhead_HundredEvidence: " + msOverheadPerCall + "ms per call"); - logger.info("Overhead was {} millis", msOverheadPerCall); - Assertions.assertTrue(msOverheadPerCall < maxOverheadPerCallMillis, - "Pipeline with share usage overhead per Process call was " + - msOverheadPerCall + "ms. Maximum permitted is " + maxOverheadPerCallMillis); + double[] rounds = measureRounds(iterations, evidenceCount); + overheadMillis.put("ShareUsageOverhead_HundredEvidence_ms", + rounds[rounds.length / 2]); + assertBelowLimit(rounds, ThousandEvidenceLimitMillis); } private static void closeAll(List data) throws Exception { diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java index eaf8670d8..1d9411cc7 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java @@ -33,6 +33,7 @@ import org.mockito.stubbing.Answer; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -291,4 +292,34 @@ public DataKey generateKey( } return result.build(); } + + /** + * A repeat less than the interval after the entry was shared is not + * shared and leaves the entry's time as it was, so the next share comes + * one interval after the last one however often the evidence is seen. A + * repeat the interval or more after it is shared, and its time is + * stored. + */ + @Test + public void ShareUsageTracker_Match_TimeChangesOnlyWhenShared() { + ShareUsageTracker tracker = new ShareUsageTracker( + new CacheConfiguration(new LruPutCache.Builder(), 1000), + NO_EXPIRY_INTERVAL_MILLIS, + evidenceKeyFilter); + long now = System.currentTimeMillis(); + + Date recent = new Date(now - NO_EXPIRY_INTERVAL_MILLIS / 2); + long sharedAt = recent.getTime(); + assertFalse(tracker.match(data, recent), + "A repeat inside the interval was shared."); + assertEquals(sharedAt, recent.getTime(), + "A repeat that was not shared changed the entry's time."); + + Date due = new Date(now - NO_EXPIRY_INTERVAL_MILLIS); + long before = System.currentTimeMillis(); + assertTrue(tracker.match(data, due), + "A repeat one interval after the entry was shared was not shared."); + assertTrue(due.getTime() >= before, + "Sharing a repeat did not store the time it was shared."); + } } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/Constants.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/Constants.java index 252924ea7..0601e4b2d 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/Constants.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/Constants.java @@ -22,6 +22,8 @@ package fiftyone.pipeline.engines; +import fiftyone.pipeline.engines.configuration.CacheConfiguration; + public class Constants { public static final String FIFTYONE_COOKIE_PREFIX = "51d_"; public static final String DEFAULT_SESSION_COOKIE_NAME = "JSESSIONID"; @@ -34,6 +36,31 @@ public class Constants { public static final boolean DEFAULT_VERIFY_IF_MODIFIED_SINCE = true; public static final boolean DEFAULT_UPDATE_ON_STARTUP = false; + /** + * Deprecated. The default of the lazy loading option, which has no + * effect. + */ + public static final boolean DEFAULT_LAZY_LOADING = false; + + /** + * Deprecated. The default timeout of the lazy loading option, which has + * no effect. + */ + public static final long DEFAULT_LAZY_LOADING_TIMEOUT_MILLIS = 1000; + + /** + * Whether a pre-packaged Pipeline caches results, unless its builder is + * told otherwise. + */ + public static final boolean DEFAULT_RESULTS_CACHE = false; + + /** + * How many results are cached when the results cache is used without a + * size, which is the cache size the caching specification gives. + */ + public static final int DEFAULT_RESULTS_CACHE_SIZE = + CacheConfiguration.defaultSize; + public enum PerformanceProfiles { LowMemory, MaxPerformance, @@ -46,23 +73,43 @@ public static class MissingPropertyMessages { public static final String PREFIX = "Property '%s' not found in data for element '%s'. "; public static final String DATA_UPGRADE_REQUIRED = - "This is because your license and/or data file " + - "does not include this property. The property is available " + - "with the %s license/data for the %s"; + "This is because your license or data file does not include " + + "this property. It is available with the %s license/data for " + + "the %s. To upgrade, see " + + "https://51degrees.com/pricing?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.engines-src-main-java-fiftyone-pipeline-engines-constants.java&utm_term=data_upgrade_required."; + /** + * The upgrade message where the data tiers the property is present + * in are not known. + */ + public static final String DATA_UPGRADE_REQUIRED_TIERS_NOT_KNOWN = + "This is because your license or data file does not include " + + "this property. To see which licenses include it, see " + + "https://51degrees.com/pricing?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.engines-src-main-java-fiftyone-pipeline-engines-constants.java&utm_term=data_upgrade_required_tiers_not_known."; public static final String PROPERTY_EXCLUDED = "This is because the property has been excluded when configuring " + "the engine."; public static final String PRODUCT_NOT_IN_CLOUD_RESOURCE = "This is because your resource key does not include access to " + - "any properties under '%s'. For more details on resource keys, " + - "see our explainer: " + - "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.engines-src-main-java-fiftyone-pipeline-engines-constants.java&utm_term=product_not_in_cloud_resource"; + "any properties under '%s'. Check or create a resource key with " + + "the properties you need in the configurator at " + + "https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.engines-src-main-java-fiftyone-pipeline-engines-constants.java&utm_term=product_not_in_cloud_resource."; public static final String PROPERTY_NOT_IN_CLOUD_RESOURCE = "This is because your resource key does not include access to " + - "this property. Properties that are included for this key under " + - "'%s' are %s. For more details on resource keys, see our " + - "explainer: " + - "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.engines-src-main-java-fiftyone-pipeline-engines-constants.java&utm_term=property_not_in_cloud_resource"; + "this property. The properties included for your key under " + + "'%s' are %s. Check or change your key in the configurator at " + + "https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.engines-src-main-java-fiftyone-pipeline-engines-constants.java&utm_term=property_not_in_cloud_resource."; + /** + * The separator of the names in a list in a message. + */ + public static final String LIST_SEPARATOR = ","; + public static final String CLOUD_REQUEST_FAILED = + "This is because an upstream cloud request failed, so no data " + + "was available to populate the property."; + /** + * The Data Key a CloudRequestFailed message names when no engine + * populates the Element Data. + */ + public static final String UNKNOWN_DATA_KEY = "Unknown"; public static final String UNKNOWN = "The reason for this is unknown. Please check that the aspect " + "and property name are correct."; diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/DataFileConfigurationBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/DataFileConfigurationBuilderBase.java index aeb9a1b29..9d7cb5818 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/DataFileConfigurationBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/DataFileConfigurationBuilderBase.java @@ -22,6 +22,8 @@ package fiftyone.pipeline.engines.configuration; +import fiftyone.pipeline.annotations.DefaultValue; +import fiftyone.pipeline.engines.Constants; import fiftyone.pipeline.engines.data.DataUpdateUrlFormatter; import fiftyone.pipeline.engines.services.DataUpdateService; @@ -71,6 +73,7 @@ public DataFileConfigurationBuilderBase() { * @param identifier the identifier to use * @return this builder instance */ + @DefaultValue("Default") @SuppressWarnings("unchecked") public TBuilder setDataFileIdentifier(String identifier) { this.identifier = identifier; @@ -83,6 +86,7 @@ public TBuilder setDataFileIdentifier(String identifier) { * @param url the URL to check for a new data file. * @return this builder instance. */ + @DefaultValue("51Degrees default, set by the engine's builder") @SuppressWarnings("unchecked") public TBuilder setDataUpdateUrl(String url) { dataUpdateUrlOverride = url; @@ -95,6 +99,7 @@ public TBuilder setDataUpdateUrl(String url) { * @param formatter the formatter to use * @return this builder instance */ + @DefaultValue("51Degrees default, set by the engine's builder") @SuppressWarnings("unchecked") public TBuilder setDataUpdateUrlFormatter( DataUpdateUrlFormatter formatter) { dataUpdateUrlFormatter = formatter; @@ -110,6 +115,7 @@ public TBuilder setDataUpdateUrlFormatter( DataUpdateUrlFormatter formatter) { * False otherwise * @return this builder instance */ + @DefaultValue(booleanValue = Constants.DEFAULT_VERIFY_MD5) @SuppressWarnings("unchecked") public TBuilder setDataUpdateVerifyMd5(boolean verify) { dataUpdateVerifyMd5 = verify; @@ -123,6 +129,7 @@ public TBuilder setDataUpdateVerifyMd5(boolean verify) { * be decompressed. False otherwise * @return this builder instance */ + @DefaultValue(booleanValue = Constants.DEFAULT_DECOMPRESS) @SuppressWarnings("unchecked") public TBuilder setDataUpdateDecompress(boolean decompress) { dataUpdateDecompress = decompress; @@ -137,6 +144,7 @@ public TBuilder setDataUpdateDecompress(boolean decompress) { * called on {@link DataUpdateService} * @return this builder instance */ + @DefaultValue(booleanValue = Constants.DEFAULT_AUTOUPDATE_ENABLED) @SuppressWarnings("unchecked") public TBuilder setAutoUpdate(boolean enabled) { autoUpdateEnabled = enabled; @@ -155,6 +163,7 @@ public TBuilder setAutoUpdate(boolean enabled) { * @param enabled the cache configuration to use * @return this builder instance */ + @DefaultValue(booleanValue = Constants.DEFAULT_WATCHER_ENABLED) @SuppressWarnings("unchecked") public TBuilder setDataFileSystemWatcher(boolean enabled) { dataFileSystemWatcherEnabled = enabled; @@ -175,6 +184,7 @@ public TBuilder setDataFileSystemWatcher(boolean enabled) { * @param pollingIntervalSeconds the number of seconds between checks * @return this builder instance */ + @DefaultValue(intValue = Constants.DATA_UPDATE_POLLING_DEFAULT) @SuppressWarnings("unchecked") public TBuilder setUpdatePollingInterval(int pollingIntervalSeconds) { updatePollingIntervalSeconds = pollingIntervalSeconds; @@ -194,6 +204,7 @@ public TBuilder setUpdatePollingInterval(int pollingIntervalSeconds) { * @param pollingIntervalMillis the time between checks * @return this builder instance */ + @DefaultValue(intValue = Constants.DATA_UPDATE_POLLING_DEFAULT * 1000) @SuppressWarnings("unchecked") public TBuilder setUpdatePollingInterval(long pollingIntervalMillis) { if (pollingIntervalMillis / 1000 > Integer.MAX_VALUE) { @@ -212,6 +223,7 @@ public TBuilder setUpdatePollingInterval(long pollingIntervalMillis) { * polling interval * @return this builder instance */ + @DefaultValue(intValue = Constants.DATA_UPDATE_RANDOMISATION_DEFAULT * 1000) @SuppressWarnings("unchecked") public TBuilder setUpdateRandomisationMax(long maximumDeviationMillis){ if (maximumDeviationMillis / 1000 > Integer.MAX_VALUE) { @@ -228,6 +240,7 @@ public TBuilder setUpdateRandomisationMax(long maximumDeviationMillis){ * @param enabled whether to use the If-Modified-Since header * @return this builder instance */ + @DefaultValue(booleanValue = Constants.DEFAULT_VERIFY_IF_MODIFIED_SINCE) @SuppressWarnings("unchecked") public TBuilder setVerifyIfModifiedSince(boolean enabled) { dataUpdateVerifyModifiedSince = enabled; @@ -241,6 +254,7 @@ public TBuilder setVerifyIfModifiedSince(boolean enabled) { * this file * @return this builder * */ + @DefaultValue("No license keys") @SuppressWarnings("unchecked") public TBuilder setDataUpdateLicenseKey(String key) { if (key == null) { @@ -265,6 +279,7 @@ public TBuilder setDataUpdateLicenseKey(String key) { * latest copy of the file * @return this builder */ + @DefaultValue(booleanValue = Constants.DEFAULT_UPDATE_ON_STARTUP) @SuppressWarnings("unchecked") public TBuilder setUpdateOnStartup(boolean enabled) { updateOnStartup = enabled; @@ -276,6 +291,7 @@ public TBuilder setUpdateOnStartup(boolean enabled) { * @param keys 51Degrees license keys * @return this builder * */ + @DefaultValue("No license keys") @SuppressWarnings("unchecked") public TBuilder setDataUpdateLicenseKeys(String[] keys) { Collections.addAll(licenseKeys, keys); diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/LazyLoadingConfiguration.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/LazyLoadingConfiguration.java index b60144ca9..196fb5402 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/LazyLoadingConfiguration.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/LazyLoadingConfiguration.java @@ -22,12 +22,14 @@ package fiftyone.pipeline.engines.configuration; -import fiftyone.pipeline.engines.exceptions.LazyLoadTimeoutException; - import java.util.concurrent.ExecutorService; /** - * Used to store configuration values relating to lazy loading. + * Deprecated, and has no effect. Lazy loading is kept as a public name so + * that existing code and configuration still build and run, but every Engine + * processes on the thread that processes the Flow Data, whatever is set + * here. Long-running work is a lazy property of an Engine's own Element Data + * instead. * @see Specification */ public class LazyLoadingConfiguration { @@ -37,12 +39,7 @@ public class LazyLoadingConfiguration { /** * Construct a new instance using the default {@link ExecutorServiceFactory}. - * @param propertyTimeoutMillis the timeout in milliseconds to use when - * waiting for processing to complete in order - * to retrieve property values. If the timeout - * is exceeded then a - * {@link LazyLoadTimeoutException} - * will be thrown + * @param propertyTimeoutMillis kept, but not used */ public LazyLoadingConfiguration(int propertyTimeoutMillis) { this(propertyTimeoutMillis, new ExecutorServiceFactoryDefault()); @@ -50,14 +47,8 @@ public LazyLoadingConfiguration(int propertyTimeoutMillis) { /** * Construct a new instance. - * @param propertyTimeoutMillis the timeout in milliseconds to use when - * waiting for processing to complete in order - * to retrieve property values. If the timeout - * is exceeded then a - * {@link LazyLoadTimeoutException} - * will be thrown - * @param factory the {@link ExecutorServiceFactory} to use when processing - * in order to retrieve property values + * @param propertyTimeoutMillis kept, but not used + * @param factory kept, but no Engine asks it for an executor */ public LazyLoadingConfiguration( int propertyTimeoutMillis, @@ -67,9 +58,7 @@ public LazyLoadingConfiguration( } /** - * Get the timeout in milliseconds to use when waiting for processing to - * complete in order to retrieve property values. If the timeout is exceeded - * then a {@link LazyLoadTimeoutException} will be thrown. + * Get the timeout that was set, which is not used. * @return timeout in milliseconds */ public int getPropertyTimeoutMillis() { @@ -77,8 +66,8 @@ public int getPropertyTimeoutMillis() { } /** - * Get an {@link ExecutorService} to use when processing in order to - * retrieve property values. + * Create an {@link ExecutorService} from the factory that was set. No + * Engine calls this, as lazy loading has no effect. * @return new {@link ExecutorService} */ public ExecutorService getExecutorService() { diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectData.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectData.java index b8fdafcb4..cee1b0bdd 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectData.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectData.java @@ -42,10 +42,10 @@ public interface AspectData extends ElementData { List> getEngines(); /** - * If the engine is configured for lazy loading, this property returns a - * {@link Future} that will complete once the engine has finished - * processing. Otherwise, it will be null. - * @return lazy loading future or null + * Deprecated. Lazy loading has no effect, so every Engine has finished + * processing when the Flow Data's process call returns, and the + * {@link Future} this returns is already done. + * @return a future that is already done */ Future getProcessFuture(); } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectDataBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectDataBase.java index 6f1ff15ae..565d692c3 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectDataBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectDataBase.java @@ -26,7 +26,6 @@ import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.data.FlowError; import fiftyone.pipeline.core.data.TryGetResult; - import fiftyone.pipeline.engines.exceptions.LazyLoadTimeoutException; import fiftyone.pipeline.engines.exceptions.PropertyMissingException; import fiftyone.pipeline.engines.flowelements.AspectEngine; import fiftyone.pipeline.engines.services.MissingPropertyResult; @@ -53,6 +52,11 @@ public abstract class AspectDataBase extends ElementDataBase implements AspectDa private final Map, Future> processFutures; + /** + * Whether the cloud request for the Flow Data failed. + */ + private volatile boolean cloudRequestFailed = false; + /** * Constructs a new instance with a non-thread-safe, case-insensitive * {@link Map} as the underlying storage. @@ -171,26 +175,52 @@ public Object get(long timeout, TimeUnit unit) }; } + /** + * Record that the cloud request for the Flow Data failed, so that a + * property this instance does not hold is missing with the + * CloudRequestFailed reason. + * @throws IllegalStateException if the Flow Element has finished + */ + public void setCloudRequestFailed() { + checkWriting(); + cloudRequestFailed = true; + } + /** * Add an engine to the list of engines which have generated the data within * this instance. * @param engine engine adding data + * @throws IllegalStateException if the Flow Element has finished */ public void addEngine(AspectEngine engine) { + checkWriting(); engines.add(engine); } /** - * Add a callable which will run a {@link AspectEngine#process(FlowData)} - * method to populate this instance. The property accessors will only - * complete once all such tasks have completed. + * Keep only the engine that made this instance, and forget the process + * tasks and failed cloud request of the request it served before. + */ + @Override + protected void onReuse() { + super.onReuse(); + engines.subList(1, engines.size()).clear(); + processFutures.clear(); + cloudRequestFailed = false; + } + + /** + * Deprecated. Runs the callable on the calling thread, as lazy loading + * has no effect. An exception it throws is given to each later read of a + * property value, as before. * @param runnable processing runnable + * @throws IllegalStateException if the Flow Element has finished */ public void addProcessCallable(ProcessCallable runnable) { - Future future = runnable.engine.getExecutor().submit(runnable); - processFutures.put( - runnable.engine, - future); + checkWriting(); + FutureTask task = new FutureTask<>(runnable); + task.run(); + processFutures.put(runnable.engine, task); } @Override @@ -199,6 +229,18 @@ public Map asKeyMap() { return super.asKeyMap(); } + /** + * Get the value stored for the key as it is now, without waiting on the + * callables added through {@link #addProcessCallable} and without the + * checks against the {@link MissingPropertyService}. The value is null + * when nothing has been stored for the key. + * @param key the key to read + * @return the value stored for the key, or null + */ + protected Object getWithoutWaiting(String key) { + return super.asKeyMap().get(key); + } + /** * Gets the value stored using the specified key with full checks * against the {@link MissingPropertyService}. @@ -221,15 +263,20 @@ protected T getAs(String key, Class type, Class... parameterisedTypes) } TryGetResult result; List errors; - if (anyLazyLoaded(engines) == false || + if (processFutures.isEmpty() || (errors = waitOnAllProcessFutures()).size() == 0) { result = tryGetValue(key, type, parameterisedTypes); if (result.hasValue() == false && missingPropertyService != null) { // If there was no entry for the key then use the missing // property service to find out why. - MissingPropertyResult missingReason = missingPropertyService - .getMissingPropertyReason(key, engines); + MissingPropertyResult missingReason = cloudRequestFailed ? + missingPropertyService.getCloudRequestFailedReason( + key, + engines) : + missingPropertyService.getMissingPropertyReason( + key, + engines); logger.debug("Property '" + key + "' missing from " + "aspect data '" + getClass().getName() + "'-'" + hashCode() + "'. " + missingReason.getReason()); @@ -241,35 +288,17 @@ protected T getAs(String key, Class type, Class... parameterisedTypes) } else { Throwable e; if (errors.size() == 1) { + // A callable added through addProcessCallable threw. e = errors.get(0).getThrowable(); - if (e instanceof CancellationException) { - // The property is being lazy loaded but been canceled, so - // pass the exception up. - throw (CancellationException)e; - } - else if (e instanceof TimeoutException) { - // The property is being lazy loaded but has timed out - // or been canceled so throw the appropriate exception. - throw new LazyLoadTimeoutException( - "Failed to retrieve property '" + key + "' " + - "because the processing for engine(s) " + - stringJoin(getDistinctEngineNames(), ", ") + - " took longer than the specified timeout.", - e); - } - else { - // The property is being lazy loaded but an error - // occurred in the engine's process method - throw new RuntimeException( - "Failed to retrieve property '" + key + "' " + - "because processing threw an exception in engine(s) " + - stringJoin(getDistinctEngineNames(), ", ") + ".", - e); - } + throw new RuntimeException( + "Failed to retrieve property '" + key + "' " + + "because processing threw an exception in engine(s) " + + stringJoin(getDistinctEngineNames(), ", ") + ".", + e); } else { - // The property is being lazy loaded but multiple errors have - // occurred in the engine's process method + // More than one callable added through addProcessCallable + // threw. throw new AggregateException( "Failed to retrieve property '" + key + "' " + "because processing threw multiple exceptions in engine(s) " + @@ -315,23 +344,8 @@ protected TryGetResult tryGetValue( } /** - * Returns true if any of the engines added have lazy loading configured. - * @param engines2 the engines to check - * @return true if any engines have lazy loading - */ - private static boolean anyLazyLoaded(List> engines2) { - for (AspectEngine engine : engines2) { - if (engine.getLazyLoadingConfiguration() != null) { - return true; - } - } - return false; - } - - /** - * Waits for all the process futures in {@link #processFutures} to finish - * then returns a list containing any errors which were thrown by the - * futures. + * Returns a list containing any errors thrown by the callables in + * {@link #processFutures}, each of which has already run. * @return list of errors */ private List waitOnAllProcessFutures() { @@ -339,9 +353,7 @@ private List waitOnAllProcessFutures() { for (Map.Entry, Future> entry : processFutures.entrySet()) { try { - entry.getValue().get( - entry.getKey().getLazyLoadingConfiguration().getPropertyTimeoutMillis(), - TimeUnit.MILLISECONDS); + entry.getValue().get(); } catch (Exception e) { errors.add(new FlowError.Default(e, entry.getKey())); } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/JsonLoader.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/JsonLoader.java index fc2ee0204..c4ebdf413 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/JsonLoader.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/JsonLoader.java @@ -28,6 +28,7 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.HashMap; +import java.util.Locale; import java.util.Map; /** @@ -54,12 +55,12 @@ private T loadData(Reader reader, Class type) throws IOException, IllegalAcce Map parameters = new HashMap<>(); for (String key : jsonObj.keySet()) { - parameters.put(key.toLowerCase(), jsonObj. get(key)); + parameters.put(key.toLowerCase(Locale.ROOT), jsonObj. get(key)); } for (Method method : type.getMethods()) { if (method.getName().startsWith("set")) { - String variableName = method.getName().substring("set".length()).toLowerCase(); + String variableName = method.getName().substring("set".length()).toLowerCase(Locale.ROOT); Class[] types = method.getParameterTypes(); if (types.length == 1 && parameters.containsKey(variableName)) { Object variable = parameters.get(variableName); diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/ProcessCallable.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/ProcessCallable.java index dfa18f730..aa9f94d70 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/ProcessCallable.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/ProcessCallable.java @@ -27,9 +27,10 @@ import java.util.concurrent.Callable; /** - * A {@link Callable} implementation which stores an {@link AspectEngine} to be - * used by the {@link Callable#call()} method. The {@link Callable#call()} - * method is left to be implemented anonymously when using this class. + * Deprecated. A {@link Callable} implementation which stores an + * {@link AspectEngine} to be used by the {@link Callable#call()} method. + * {@link AspectDataBase#addProcessCallable(ProcessCallable)} runs it on the + * calling thread, as lazy loading has no effect. */ public abstract class ProcessCallable implements Callable { diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/ProvidesValuesAsString.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/ProvidesValuesAsString.java new file mode 100644 index 000000000..292632357 --- /dev/null +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/ProvidesValuesAsString.java @@ -0,0 +1,51 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.data; + +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; + +/** + * Implemented by Element Data that holds Property values as strings and + * parses them into the type each Property declares, so a caller can read the + * string the data holds as well as the parsed value. + *

    + * A parsed value cannot always say what the data holds. "Unknown" in a + * Boolean Property, for example, is parsed to false, which cannot be told + * apart from a false the data really holds. Test for this interface at run + * time, and use the typed accessors where it is absent. + * @see Specification + */ +public interface ProvidesValuesAsString { + + /** + * Get the string stored for a Property, as it was stored. The string is + * never made from a parsed value, and reading it changes nothing the + * other accessors return. + * @param propertyName the name of the Property, in any case + * @return the stored string, or the reason there is no value, as for any + * other accessor + * @throws PropertyMissingException where the Property is not in the + * results, in the same way as any other accessor + */ + AspectPropertyValue getStoredValueAsString(String propertyName); +} diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/exceptions/LazyLoadTimeoutException.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/exceptions/LazyLoadTimeoutException.java index 200dc6f90..44293bf7a 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/exceptions/LazyLoadTimeoutException.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/exceptions/LazyLoadTimeoutException.java @@ -22,11 +22,11 @@ package fiftyone.pipeline.engines.exceptions; -import fiftyone.pipeline.core.data.ElementData; /** - * Timeout exception thrown if the lazy loading of a property value from an - * {@link ElementData} times out. + * Deprecated. Lazy loading has no effect, so nothing waits on a timeout and + * this is no longer thrown. It is kept so that code that catches it still + * builds. */ public class LazyLoadTimeoutException extends RuntimeException { diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngine.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngine.java index 356dc5ae8..15debec1b 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngine.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngine.java @@ -65,21 +65,25 @@ public interface AspectEngine< String getDataSourceTier(); /** - * Configure lazy loading of results. - * @param configuration the configuration to use + * Deprecated, and has no effect. Every Engine processes on the thread + * that processes the Flow Data, whether this is set or not. Long-running + * work is a lazy property of the Engine's own Element Data instead. + * @param configuration kept, but not used + * @see Specification */ void setLazyLoading(LazyLoadingConfiguration configuration); /** - * Get the lazy loading configuration used for loading of results. - * @return lazy loading configuration + * Deprecated. Get the lazy loading configuration that was set, which has + * no effect. + * @return lazy loading configuration, or null if none was set */ LazyLoadingConfiguration getLazyLoadingConfiguration(); /** - * Get the executor service to use when starting processing threads which - * are lazily loaded. - * @return an {@link ExecutorService} + * Deprecated. Lazy loading has no effect, so an Engine starts no + * processing threads. + * @return null, as no executor is used */ ExecutorService getExecutor(); } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBase.java index 89a545be1..b828c38d5 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBase.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.engines.flowelements; +import fiftyone.pipeline.core.data.ElementDataBase; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.data.factories.ElementDataFactory; import fiftyone.pipeline.core.flowelements.FlowElementBase; @@ -32,7 +33,6 @@ import fiftyone.pipeline.engines.data.AspectData; import fiftyone.pipeline.engines.data.AspectDataBase; import fiftyone.pipeline.engines.data.AspectPropertyMetaData; -import fiftyone.pipeline.engines.data.ProcessCallable; import fiftyone.pipeline.engines.services.MissingPropertyService; import fiftyone.pipeline.util.Types; import org.slf4j.Logger; @@ -40,7 +40,6 @@ import java.io.IOException; import java.util.List; import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; /** * Base class for {@link AspectEngine}s to extend. This implements the data @@ -56,8 +55,6 @@ public abstract class AspectEngineBase< private LazyLoadingConfiguration lazyLoadingConfiguration = null; - private ExecutorService executor = null; - protected MissingPropertyService missingPropertyService; protected FlowCache cache = null; @@ -97,10 +94,17 @@ public void setCache(FlowCache cache) { cache.setFlowElement(this); } + /** + * Deprecated, and has no effect. The configuration is kept so that it can + * be read back, but the Engine always processes on the thread that + * processes the Flow Data, and no executor is created. Long-running work + * is a lazy property of the Engine's own Element Data instead. + * @param configuration ignored apart from being kept + * @see Specification + */ @Override public void setLazyLoading(LazyLoadingConfiguration configuration) { this.lazyLoadingConfiguration = configuration; - this.executor = lazyLoadingConfiguration.getExecutorService(); } @Override @@ -108,9 +112,13 @@ public LazyLoadingConfiguration getLazyLoadingConfiguration() { return lazyLoadingConfiguration; } + /** + * Deprecated. Lazy loading has no effect, so no executor is created. + * @return always null + */ @Override public ExecutorService getExecutor() { - return executor; + return null; } /** @@ -135,6 +143,38 @@ protected final void processInternal(FlowData flowData) throws Exception { processWithCache(flowData); } + /** + * Whether a processed result may be stored in the cache. A result that + * holds a value created for the one request that asked for it, such as a + * 51Did, must be neither stored nor served. See the + * caching specification: + * "A cache MUST NOT store a result that holds such a value, and MUST NOT + * serve one that it finds." + *

    + * It is asked about a processed result before it is stored. + * {@link #isServable} is asked about a result found in the cache before + * it is served. + * @param data the result to check + * @return true, unless an engine overrides this to keep a result out of + * the cache + */ + protected boolean isCacheable(TData data) { + return true; + } + + /** + * Whether a result found in the cache may be served. By default the + * answer is {@link #isCacheable}'s. An engine whose result is still being + * worked out by a lazy task when it is stored overrides this, so that a + * result is served only once its task has finished and the finished + * result may be served. + * @param data the result found in the cache + * @return true if the result may be served + */ + protected boolean isServable(TData data) { + return isCacheable(data); + } + /** * Private method that checks if the result is already in the cache or not. * If it is then the result is added to 'data', if not then @@ -151,7 +191,8 @@ private void processWithCache(final FlowData flowData) throws Exception { Object cacheResultsObject = cache.get(flowData); if (cacheResultsObject != null && getTypedDataKey().getType().isAssignableFrom( - cacheResultsObject.getClass())) { + cacheResultsObject.getClass()) && + isServable((TData)cacheResultsObject)) { // This is checked. //noinspection unchecked cacheResult = (TData)cacheResultsObject; @@ -169,30 +210,10 @@ private void processWithCache(final FlowData flowData) throws Exception { ((AspectDataBase) aspectData).addEngine(this); } - // Start the engine processing - if (lazyLoadingConfiguration != null) { - // If lazy loading is configured then create a task - // to do the processing and assign the task to the - // aspect data property. - ((AspectDataBase) aspectData).addProcessCallable( - new ProcessCallable(this) { - @Override - public Void call() throws Exception { - processEngine(flowData, aspectData); - return null; - } - } - ); - } - else { - // If not lazy loading, just start processing. - processEngine(flowData, aspectData); - } - // If there is a cache then add the result - // of processing to the cache. - if (cache != null) { - cache.put(flowData, flowData.getFromElement(this)); - } + // Process on this thread, whether or not the deprecated lazy + // loading option is set, then cache the result if it may be. + processEngine(flowData, aspectData); + putInCache(flowData, aspectData); } else { // We have a result from the cache so add it // into the flow data. @@ -200,6 +221,22 @@ public Void call() throws Exception { } } + /** + * Add a processed result to the cache, if there is one and the result + * may be cached. A result that is stored is kept out of its pool, + * because the cache holds it after this Flow Data is closed. + * @param flowData the {@link FlowData} whose Evidence makes the key + * @param aspectData the processed result + */ + private void putInCache(FlowData flowData, TData aspectData) { + if (cache != null && isCacheable(aspectData)) { + if (aspectData instanceof ElementDataBase) { + ((ElementDataBase) aspectData).keepFromPool(); + } + cache.put(flowData, aspectData); + } + } + @Override protected void managedResourcesCleanup() { if (cache != null) { @@ -209,17 +246,5 @@ protected void managedResourcesCleanup() { logger.warn("Exception closing the cache", e); } } - if (executor != null) { - executor.shutdown(); - try { - if (executor.awaitTermination( - lazyLoadingConfiguration.getPropertyTimeoutMillis(), - TimeUnit.MILLISECONDS) == false) { - executor.shutdownNow(); - } - } catch (InterruptedException e) { - executor.shutdownNow(); - } - } } } \ No newline at end of file diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBase.java index aaba5f317..54ff3eeb4 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBase.java @@ -95,6 +95,7 @@ protected List getProperties() { * @param set The properties that we want the engine to populate * @return this builder */ + @DefaultValue("All properties") @SuppressWarnings("unchecked") public TBuilder setProperties(Set set) { for (AspectPropertyMetaData property : set) { @@ -124,6 +125,7 @@ public TBuilder setProperty(String s) { * @param properties The properties that we want the engine to populate * @return this builder */ + @DefaultValue("All properties") @SuppressWarnings("unchecked") public TBuilder setProperties(List properties) { for (String property : properties) { @@ -136,14 +138,19 @@ public TBuilder setProperties(List properties) { * Configure the properties that the engine will populate in the response. *

    * By default, all properties will be populated. - * @param properties The properties that we want the engine to populate + * @param properties The properties that we want the engine to populate, + * separated by commas as a configuration file gives a + * list, with any spaces round each name ignored * @return this builder */ @DefaultValue("All properties") @SuppressWarnings("unchecked") public TBuilder setProperties(String properties) { - for (String property : properties.split("\\s,\\s")) { - tryAddProperty(property); + for (String property : properties.split(",")) { + String name = property.trim(); + if (name.isEmpty() == false) { + tryAddProperty(name); + } } return (TBuilder) this; } @@ -154,6 +161,7 @@ public TBuilder setProperties(String properties) { * @param aspectProperty the property that we want the engine to populate * @return this builder */ + @DefaultValue("All properties") @SuppressWarnings("unchecked") public TBuilder setProperty(AspectPropertyMetaData aspectProperty) { tryAddProperty(aspectProperty.getName()); @@ -161,10 +169,14 @@ public TBuilder setProperty(AspectPropertyMetaData aspectProperty) { } /** - * Configure lazy loading of results. - * @param configuration the configuration to use + * Deprecated, and has no effect. The Engine processes on the thread that + * processes the Flow Data, whether this is set or not. Long-running work + * is a lazy property of the Engine's own Element Data instead. + * @param configuration kept, but not used * @return this builder + * @see Specification */ + @DefaultValue("No lazy loading") @SuppressWarnings("unchecked") public TBuilder setLazyLoading(LazyLoadingConfiguration configuration) { this.lazyLoadingConfig = configuration; @@ -177,6 +189,7 @@ public TBuilder setLazyLoading(LazyLoadingConfiguration configuration) { * @param cacheConfiguration the cache configuration to use * @return this builder */ + @DefaultValue("No results cache") @SuppressWarnings("unchecked") public TBuilder setCache(CacheConfiguration cacheConfiguration) { this.cacheConfig = cacheConfiguration; diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudAspectEngine.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudAspectEngine.java index f08018998..ff9caed9d 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudAspectEngine.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudAspectEngine.java @@ -37,4 +37,13 @@ public interface CloudAspectEngine< TData extends AspectData, TProperty extends AspectPropertyMetaData> extends AspectEngine { + + /** + * Whether the request for the list of properties the cloud service + * offers failed, so that this engine's property metadata is not loaded. + * @return true if the request for the property list failed + */ + default boolean propertyListRequestFailed() { + return false; + } } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudPipelineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudPipelineBuilderBase.java index fed30283a..234f0099c 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudPipelineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudPipelineBuilderBase.java @@ -22,8 +22,13 @@ package fiftyone.pipeline.engines.flowelements; +import fiftyone.pipeline.annotations.DefaultValue; import org.slf4j.ILoggerFactory; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + /** * Base class for pipeline builders that will produce a pipeline with specific * flow elements. @@ -45,6 +50,8 @@ public abstract class CloudPipelineBuilderBase< protected String licenseKey = ""; + protected final List requestedProperties = new ArrayList<>(); + protected String cloudRequestOrigin = ""; /** @@ -64,6 +71,8 @@ public CloudPipelineBuilderBase(ILoggerFactory loggerFactory) { * @param url endpoint URL * @return this builder */ + @DefaultValue("https://cloud.51degrees.com/api/v4/, the Cloud Request " + + "Engine's default") @SuppressWarnings("unchecked") public TBuilder setEndPoint(String url) { if (url.endsWith("/") == false) { @@ -78,6 +87,7 @@ public TBuilder setEndPoint(String url) { * @param url data URL * @return this builder */ + @DefaultValue("The endpoint followed by {resourcekey}.json") @SuppressWarnings("unchecked") public TBuilder setDataEndpoint(String url) { this.dataEndpoint = url; @@ -90,6 +100,7 @@ public TBuilder setDataEndpoint(String url) { * @param propertiesEndpoint properties URL * @return this builder */ + @DefaultValue("The endpoint followed by accessibleproperties") @SuppressWarnings("unchecked") public TBuilder setPropertiesEndpoint(String propertiesEndpoint) { this.propertiesEndpoint = propertiesEndpoint; @@ -102,6 +113,7 @@ public TBuilder setPropertiesEndpoint(String propertiesEndpoint) { * @param evidenceKeysEndpoint evidence keys URL * @return this builder */ + @DefaultValue("The endpoint followed by evidencekeys") @SuppressWarnings("unchecked") public TBuilder setEvidenceKeysEndpoint(String evidenceKeysEndpoint) { this.evidenceKeysEndpoint = evidenceKeysEndpoint; @@ -113,6 +125,8 @@ public TBuilder setEvidenceKeysEndpoint(String evidenceKeysEndpoint) { * @param key resource key * @return this builder */ + @DefaultValue("No default - a resource key or a license key must be " + + "supplied") @SuppressWarnings("unchecked") public TBuilder setResourceKey(String key) { this.resourceKey = key; @@ -120,16 +134,55 @@ public TBuilder setResourceKey(String key) { } /** - * Set the license key to be used when calling the endpoints. + * Set the license key to be used when calling the endpoints. A license + * key used without a resource key needs the properties wanted, named + * with {@link #setRequestedProperties(List)}. Beside a resource key it + * adds the requested properties the license grants to what the resource + * key carries. * @param key license key * @return this builder */ + @DefaultValue("No default") @SuppressWarnings("unchecked") public TBuilder setLicenseKey(String key) { this.licenseKey = key; return (TBuilder)this; } + /** + * Name the properties each request asks the cloud service for, as fully + * qualified names such as {@code device.ismobile}. Required with a + * license key alone. Beside a resource key the cloud service adds them + * to the properties the resource key carries. + * @param properties the fully qualified property names + * @return this builder + */ + @DefaultValue("None - required with a license key alone") + @SuppressWarnings("unchecked") + public TBuilder setRequestedProperties(List properties) { + if (properties == null) { + throw new IllegalArgumentException( + "properties must not be null"); + } + requestedProperties.addAll(properties); + return (TBuilder)this; + } + + /** + * As {@link #setRequestedProperties(List)}, taking the names as one + * comma separated string. + * @param properties comma separated fully qualified property names + * @return this builder + */ + @DefaultValue("None - required with a license key alone") + public TBuilder setRequestedProperties(String properties) { + if (properties == null) { + throw new IllegalArgumentException( + "properties must not be null"); + } + return setRequestedProperties(Arrays.asList(properties.split(","))); + } + /** * The value to set for the Origin header when making requests * to the cloud service. @@ -141,6 +194,7 @@ public TBuilder setLicenseKey(String key) { * @param cloudRequestOrigin the value to set the origin header to * @return this builder */ + @DefaultValue("None") @SuppressWarnings("unchecked") public TBuilder setCloudRequestOrigin(String cloudRequestOrigin) { this.cloudRequestOrigin = cloudRequestOrigin; diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java index 879f55f44..f743cd6a6 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java @@ -174,6 +174,8 @@ public TBuilder setTempDirPath(String dirPath) { * @param profile the performance profile for the engine to use * @return this engine builder instance */ + @DefaultValue("Set by each engine's builder, which gives it on its own " + + "setPerformanceProfile") public abstract TBuilder setPerformanceProfile(Constants.PerformanceProfiles profile); @Override diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/PrePackagedPipelineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/PrePackagedPipelineBuilderBase.java index 92188b7f2..83ac76a40 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/PrePackagedPipelineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/PrePackagedPipelineBuilderBase.java @@ -23,6 +23,7 @@ package fiftyone.pipeline.engines.flowelements; import fiftyone.pipeline.core.flowelements.PipelineBuilderBase; +import fiftyone.pipeline.engines.Constants; import org.slf4j.ILoggerFactory; import org.slf4j.LoggerFactory; @@ -35,13 +36,14 @@ public abstract class PrePackagedPipelineBuilderBase > extends PipelineBuilderBase { - protected boolean lazyLoading = false; + protected boolean lazyLoading = Constants.DEFAULT_LAZY_LOADING; - protected boolean resultsCache = false; + protected boolean resultsCache = Constants.DEFAULT_RESULTS_CACHE; - protected long lazyLoadingTimeoutMillis = 1000; + protected long lazyLoadingTimeoutMillis = + Constants.DEFAULT_LAZY_LOADING_TIMEOUT_MILLIS; - protected int resultsCacheSize = 1000; + protected int resultsCacheSize = Constants.DEFAULT_RESULTS_CACHE_SIZE; /** * Construct a new instance using the default {@link ILoggerFactory} @@ -63,8 +65,10 @@ public PrePackagedPipelineBuilderBase(ILoggerFactory loggerFactory) { } /** - * Enable lazy loading of results. Uses a default timeout of 5 seconds. + * Deprecated, and has no effect. Every Engine processes on the thread + * that processes the Flow Data, whether this is set or not. * @return this builder instance + * @see Specification */ @SuppressWarnings("unchecked") public TBuilder useLazyLoading() { @@ -73,10 +77,12 @@ public TBuilder useLazyLoading() { } /** - * Enable lazy loading of results. - * @param timeoutMillis the timeout to use when attempting to access - * lazy-loaded values. Default is 5 seconds + * Deprecated, and has no effect. Every Engine processes on the thread + * that processes the Flow Data, whether this is set or not, so the + * timeout is not used. + * @param timeoutMillis kept, but not used * @return this builder + * @see Specification */ @SuppressWarnings("unchecked") public TBuilder useLazyLoading(long timeoutMillis) { @@ -86,7 +92,8 @@ public TBuilder useLazyLoading(long timeoutMillis) { } /** - * Enable caching of results. Uses a default cache size of 1000. + * Enable caching of results, with the size + * {@link Constants#DEFAULT_RESULTS_CACHE_SIZE}. * @return this builder */ @SuppressWarnings("unchecked") @@ -98,7 +105,7 @@ public TBuilder useResultsCache() { /** * Enable caching of results. * @param size the maximum number of results to hold in the device detection - * cache. Default is 1000 + * cache * @return this builder */ @SuppressWarnings("unchecked") diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/SingleFileAspectEngineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/SingleFileAspectEngineBuilderBase.java index 84df5a4f9..d389e1600 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/SingleFileAspectEngineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/SingleFileAspectEngineBuilderBase.java @@ -242,6 +242,7 @@ public TBuilder setDataFileSystemWatcher(boolean enabled) { * @return this builder */ @Deprecated + @DefaultValue(intValue = Constants.DATA_UPDATE_POLLING_DEFAULT) @SuppressWarnings("unchecked") public TBuilder setUpdatePollingInterval(int pollingIntervalSeconds) { dataFileBuilder.setUpdatePollingInterval(pollingIntervalSeconds); @@ -277,6 +278,7 @@ public TBuilder setUpdatePollingIntervalSeconds(int pollingIntervalSeconds) { * @return this builder */ @Deprecated + @DefaultValue(intValue = Constants.DATA_UPDATE_POLLING_DEFAULT * 1000) @SuppressWarnings("unchecked") public TBuilder setUpdatePollingInterval(long pollingIntervalMillis) { dataFileBuilder.setUpdatePollingInterval(pollingIntervalMillis); @@ -308,6 +310,7 @@ public TBuilder setUpdatePollingIntervalMillis(long pollingIntervalMillis) { * @return this builder */ @Deprecated + @DefaultValue(intValue = Constants.DATA_UPDATE_RANDOMISATION_DEFAULT * 1000) @SuppressWarnings("unchecked") public TBuilder setUpdateRandomisationMax(long maximumDeviationMillis) { dataFileBuilder.setUpdateRandomisationMax(maximumDeviationMillis); @@ -324,7 +327,7 @@ public TBuilder setUpdateRandomisationMax(long maximumDeviationMillis) { * polling interval * @return this builder */ - @DefaultValue(intValue = Constants.DATA_UPDATE_RANDOMISATION_DEFAULT) + @DefaultValue(intValue = Constants.DATA_UPDATE_RANDOMISATION_DEFAULT * 1000) @SuppressWarnings("unchecked") public TBuilder setUpdateRandomisationMaxMillis(long maximumDeviationMillis) { dataFileBuilder.setUpdateRandomisationMax(maximumDeviationMillis); diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java index 2ee0231c0..eeb93a9c2 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java @@ -129,7 +129,11 @@ enum AutoUpdateStatus { // Refreshing the engine with the new data caused an error to occur. AUTO_UPDATE_REFRESH_FAILED, // An operation could not be completed within the expected time. - AUTO_UPDATE_TIMEOUT + AUTO_UPDATE_TIMEOUT, + // The data file checking for updates has no temporary file path. + AUTO_UPDATE_TEMP_PATH_NOT_SET, + // A check failed with an error that no other status covers. + AUTO_UPDATE_UNKNOWN_ERROR } class DataUpdateCompleteArgs { diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java index 398bfd9b9..2c836e0bf 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java @@ -67,6 +67,13 @@ */ public class DataUpdateServiceDefault implements DataUpdateService { + /** + * The log message of a failed check for a data update, with the error + * that says why attached. + */ + private static final String CHECK_FAILED = + "Error during check for data update"; + /** * The query parameter of a data update URL that holds the license keys, * as the 51Degrees URL formatter writes it. @@ -209,6 +216,10 @@ public AutoUpdateStatus checkForUpdate(OnPremiseAspectEngineexception-handling.md line 83: + * "When errors do occur, they will be logged and MUST NOT cause the + * process to crash." An error of the virtual machine itself, such as + * running out of memory, is logged and then thrown on. + * @param what what the task does, for the log + * @param task the task + * @return the wrapped task + */ + private Runnable logErrors(final String what, final Runnable task) { + return new Runnable() { + @Override + public void run() { + try { + task.run(); + } catch (Throwable t) { + logger.error("Unexpected error in the background " + + what + ".", t); + if (t instanceof VirtualMachineError) { + throw (VirtualMachineError) t; + } + } + } + }; + } + private AutoUpdateStatus checkForUpdate(Object state, boolean manualUpdate) { + AspectEngineDataFile dataFile = + state instanceof AspectEngineDataFileDefault ? + (AspectEngineDataFileDefault) state : + null; + AutoUpdateStatus result; + try { + result = runCheck(state, manualUpdate); + } catch (RuntimeException | Error ex) { + report(state, AutoUpdateStatus.AUTO_UPDATE_UNKNOWN_ERROR, dataFile); + throw ex; + } + report(state, result, dataFile); + return result; + } + + /** + * Run a check started by update polling. A failure of the check is + * logged, reported as {@link AutoUpdateStatus#AUTO_UPDATE_UNKNOWN_ERROR}, + * and the next check is scheduled, so a later data file is still + * applied. A failure in an update callback is logged. The timer's task + * would otherwise lose either one. + * + * @param dataFile the data file to check + */ + private void checkForUpdateOnTimer(AspectEngineDataFileDefault dataFile) { + AutoUpdateStatus result; + try { + result = runCheck(dataFile, false); + } catch (RuntimeException | Error ex) { + logUnhandledError(dataFile, ex); + result = AutoUpdateStatus.AUTO_UPDATE_UNKNOWN_ERROR; + scheduleNextCheck(dataFile); + } + reportOnTimer(dataFile, result, dataFile); + } + + /** + * The steps of {@link #checkForUpdate(Object, boolean)}, without the + * report to the update callbacks. + * + * @param state the data file meta data + * @param manualUpdate true if the check was called programmatically + * @return the status of the check + */ + private AutoUpdateStatus runCheck(Object state, boolean manualUpdate) { AutoUpdateStatus result = AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS; final AspectEngineDataFileDefault dataFile = @@ -253,11 +338,7 @@ private AutoUpdateStatus checkForUpdate(Object state, boolean manualUpdate) { if (newDataAvailable == false && dataFile.getConfiguration().getDataUpdateUrl() != null && dataFile.getConfiguration().getDataUpdateUrl().isEmpty() == false) { - // A failed update is tried again only when the timer is on. - result = checkForUpdateFromUrl( - dataFile, - manualUpdate == false && - dataFile.getAutomaticUpdatesEnabled()); + result = checkForUpdateFromUrl(dataFile); newDataAvailable = result == AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS || result == AutoUpdateStatus.AUTO_UPDATE_SUCCESS; @@ -289,12 +370,12 @@ private AutoUpdateStatus checkForUpdate(Object state, boolean manualUpdate) { if (dataFile.getFuture() != null) { dataFile.getFuture().cancel(true); dataFile.setFuture(futureFactory.schedule( - new Runnable() { + logErrors("update check", new Runnable() { @Override public void run() { - checkForUpdate(dataFile, false); + checkForUpdateOnTimer(dataFile); } - }, + }), getIntervalMillis(dataFile.getConfiguration()))); } } @@ -313,11 +394,6 @@ public void run() { } } - for (OnUpdateComplete onUpdateComplete : onUpdateCompleteList) { - onUpdateComplete.call(state, new DataUpdateCompleteArgs( - result, - dataFile)); - } return result; } @@ -384,9 +460,7 @@ private AutoUpdateStatus updatedFileAvailable(AspectEngineDataFile dataFile) { * @param sender the {@link AspectEngineDataFile} */ private void dataFileUpdated(Object sender) { - AutoUpdateStatus status = AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS; // Get the associated update configuration - AspectEngineDataFile dataFile = null; for (AspectEngineDataFile current : configurations) { if (current.equals(sender)) { @@ -395,6 +469,25 @@ private void dataFileUpdated(Object sender) { } } + AutoUpdateStatus status; + try { + status = applyWatchedFile(dataFile); + } catch (RuntimeException | Error ex) { + logUnhandledError(dataFile, ex); + status = AutoUpdateStatus.AUTO_UPDATE_UNKNOWN_ERROR; + } + reportOnTimer(sender, status, dataFile); + } + + /** + * Apply the data file the file watcher found changed, once however many + * events it raised. + * + * @param dataFile the registered data file, or null if there is none + * @return the status of the check + */ + private AutoUpdateStatus applyWatchedFile(AspectEngineDataFile dataFile) { + AutoUpdateStatus status = AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS; if (dataFile != null) { AspectEngineDataFileDefault aspectDataFile = (AspectEngineDataFileDefault) dataFile; // Get the creation time of the new data file @@ -433,12 +526,16 @@ private void dataFileUpdated(Object sender) { } } if (fileLockable) { - // Complete the update - status = updatedFileAvailable(dataFile); - // The refresh cancelled the polling timer, so - // register the file again to set a new one. - if (dataFile.getIsRegistered()) { - registerDataFile(dataFile); + // Complete the update. The refresh cancels the + // polling timer first, so the file is registered + // again to set a new one however the refresh + // ends, an Error included. + try { + status = updatedFileAvailable(dataFile); + } finally { + if (dataFile.getIsRegistered()) { + registerDataFile(dataFile); + } } } else if (interrupted == false) { // The polling timer is left as it was. @@ -450,9 +547,7 @@ private void dataFileUpdated(Object sender) { } } } - for (OnUpdateComplete onUpdateComplete : onUpdateCompleteList) { - onUpdateComplete.call(sender, new DataUpdateCompleteArgs(status, dataFile)); - } + return status; } @Override @@ -465,9 +560,10 @@ public AutoUpdateStatus updateFromMemory(AspectEngineDataFile dataFile, byte[] d fileWrapperFactory.build(dataFile.getDataFilePath()).getWriter() .writeBytes(data); } catch (Exception ex) { - logger.error("An error occurred when writing to " + - dataFile.getDataFilePath() + - ". Error detail: " + ex.getMessage(), ex); + logger.warn("An error occurred when writing to '" + + dataFile.getDataFilePath() + "'. The engine will be " + + "updated to use the new data but the file on disk " + + "will still contain old data.", ex); } } @@ -536,15 +632,15 @@ public void registerDataFile(final AspectEngineDataFile dataFile) { delayMillis); try { ScheduledFuture future = futureFactory.schedule( - new Runnable() { + logErrors("update check", new Runnable() { @Override public void run() { logger.debug("running checkForUpdate {}", aspectDataFile.getDataFilePath()); - checkForUpdate(aspectDataFile, false); + checkForUpdateOnTimer(aspectDataFile); logger.debug("Returning from checkForUpdate"); } - }, + }), delayMillis); logger.debug("Update check scheduled for {} millis", future.getDelay(TimeUnit.MILLISECONDS)); @@ -596,7 +692,7 @@ public void run() { aspectDataFile.getConfiguration().getPollingIntervalSeconds() * 1000L; logger.debug("Setting poll future {} millis", pollingInterval); aspectDataFile.setPollFuture(futureFactory.scheduleRepeating( - new Runnable() { + logErrors("file system watch", new Runnable() { @Override public void run() { logger.debug("Watcher is running"); @@ -619,7 +715,7 @@ public void run() { // make sure to reset the watch key or we are not called again watchKey.reset(); } - }, + }), pollingInterval)); logger.debug("Poll check scheduled for {} mins", aspectDataFile.getPollFuture().getDelay(TimeUnit.MINUTES)); @@ -799,13 +895,11 @@ private static List licenseKeysToMask( * * @param dataFile the data file to use * @param tempFile the temp file to write the data to - * @param automatic true if the update will be tried again if it fails * @return the {@link AutoUpdateStatus} value indicating the result */ private AutoUpdateStatus downloadFile( AspectEngineDataFile dataFile, - FileWrapper tempFile, - boolean automatic) { + FileWrapper tempFile) { String url = dataFile.getFormattedUrl(); String engineType = getEngineType(dataFile); @@ -820,8 +914,8 @@ private AutoUpdateStatus downloadFile( loggedUrl, engineType); HttpURLConnection connection = httpClient.connect(new URI(url.trim()).toURL()); if (connection == null) { - logger.warn(connectFailure(loggedUrl, engineType, automatic, - "No response from the data update service")); + logCheckFailure("No response from data update service at '" + + loggedUrl + "' for engine '" + engineType + "'", null); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } try { @@ -840,13 +934,12 @@ private AutoUpdateStatus downloadFile( AutoUpdateStatus downloaded; try { downloaded = downloadOkResponse( - connection, dataFile, tempFile, automatic); + connection, dataFile, tempFile, loggedUrl); } catch (Exception e) { - // An HTTP library's exception often names the - // URL it requested. - logger.warn(downloadFailure(loggedUrl, engineType, - automatic, LicenseKeyMask.maskIn( - e.getMessage(), licenseKeys)), + // A download cut short is the access error, as + // a failure to connect is. An HTTP library's + // exception often names the URL it requested. + logCheckFailure(accessFailure(loggedUrl, engineType), LicenseKeyMask.maskIn(e, licenseKeys)); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } @@ -859,8 +952,8 @@ private AutoUpdateStatus downloadFile( // Note: needed because TooManyRequests is not available // in some versions of the HttpStatusCode enum. case (429): - logger.warn(downloadFailure(loggedUrl, engineType, automatic, - "HTTP status code 429 (too many requests)")); + logCheckFailure("Too many requests to '" + loggedUrl + + "' for engine '" + engineType + "'", null); return AutoUpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS; case HttpURLConnection.HTTP_NOT_MODIFIED: logger.info("No data newer than {} found at '{}' for Engine '{}'", @@ -868,12 +961,15 @@ private AutoUpdateStatus downloadFile( engineType); return AutoUpdateStatus.AUTO_UPDATE_NOT_NEEDED; case HttpURLConnection.HTTP_FORBIDDEN: - logger.warn(downloadFailure(loggedUrl, engineType, automatic, - "HTTP status code 403 (access denied)")); + logCheckFailure("Access denied to data update service " + + "at '" + loggedUrl + "' for engine '" + + engineType + "'", null); return AutoUpdateStatus.AUTO_UPDATE_ERR_403_FORBIDDEN; default: - logger.warn(downloadFailure(loggedUrl, engineType, automatic, - "HTTP status code " + connection.getResponseCode())); + logCheckFailure("HTTP status code '" + + connection.getResponseCode() + "' from data " + + "update service at '" + loggedUrl + + "' for engine '" + engineType + "'", null); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } } finally { @@ -881,8 +977,7 @@ private AutoUpdateStatus downloadFile( } } catch (Exception e) { // An HTTP library's exception often names the URL it requested. - logger.warn(connectFailure(loggedUrl, engineType, automatic, - LicenseKeyMask.maskIn(e.getMessage(), licenseKeys)), + logCheckFailure(accessFailure(loggedUrl, engineType), LicenseKeyMask.maskIn(e, licenseKeys)); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } @@ -906,36 +1001,25 @@ private long getIfModifiedSince(AspectEngineDataFile dataFile) { return publishedMillis; } - private static String retryNote(boolean automatic) { - return automatic ? " Update will be attempted again later." : ""; - } - /** - * The warning for a failed connection. The URL and the detail are + * The message of the Errors table's row for a failure to connect to the + * data update service, or to download all of its response. The URL is * given with each license key already masked. */ - private static String connectFailure( - String loggedUrl, - String engineType, - boolean automatic, - String detail) { - return "An error occurred when connecting to " + loggedUrl + - " in order to check for data file updates for " + engineType + - "." + retryNote(automatic) + " Error detail: " + detail; + private static String accessFailure(String loggedUrl, String engineType) { + return "Error accessing data update service at '" + loggedUrl + + "' for engine '" + engineType + "'"; } /** - * The warning for a failed download. The URL and the detail are given - * with each license key already masked. + * Log an Error row of the Errors table as the specification asks, at + * Error, reading "Error during check for data update", with an error + * attached that carries the row's message and the cause when there is + * one. The message and the cause are given with each license key + * already masked. */ - private static String downloadFailure( - String loggedUrl, - String engineType, - boolean automatic, - String detail) { - return "An error occurred while downloading a data file update for " + - engineType + " from " + loggedUrl + "." + retryNote(automatic) + - " Error detail: " + detail; + private void logCheckFailure(String message, Throwable cause) { + logger.error(CHECK_FAILED, new IOException(message, cause)); } /** @@ -953,6 +1037,69 @@ private void logRefreshFailure( (tries == null ? "." : " after " + tries + " tries."), ex); } + /** + * Log the Errors table's message for an error in a check started by + * update polling or the file watcher that no other row covers, or one + * raised by an update callback, with the error attached. A Java Error, + * such as one from the native code during a refresh, is a failure of + * the check like any other. + */ + private void logUnhandledError( + AspectEngineDataFile dataFile, + Throwable ex) { + logger.error("An unhandled error occurred while checking for " + + "automatic updates for engine '" + + (dataFile == null ? "No Engine" : getEngineType(dataFile)) + + "'", ex); + } + + /** + * Report a check to each update callback in turn. An error a callback + * raises passes to the caller, and the callbacks after it are not + * called. + */ + private void report( + Object sender, + AutoUpdateStatus status, + AspectEngineDataFile dataFile) { + for (OnUpdateComplete onUpdateComplete : onUpdateCompleteList) { + onUpdateComplete.call(sender, + new DataUpdateCompleteArgs(status, dataFile)); + } + } + + /** + * Report a check started by update polling or the file watcher, and log + * an error a callback raises, as the scheduled task would lose it. + */ + private void reportOnTimer( + Object sender, + AutoUpdateStatus status, + AspectEngineDataFile dataFile) { + try { + report(sender, status, dataFile); + } catch (RuntimeException | Error ex) { + logUnhandledError(dataFile, ex); + } + } + + /** + * Clear the timer of a registered data file and register it again, + * which sets a new one, so polling goes on after a failed check. The + * timer is not interrupted, as the check may be running in it. + */ + private void scheduleNextCheck(AspectEngineDataFileDefault dataFile) { + synchronized (futureLock) { + if (dataFile.getFuture() != null) { + dataFile.getFuture().cancel(false); + dataFile.setFuture(null); + } + } + if (dataFile.getIsRegistered()) { + registerDataFile(dataFile); + } + } + private String getEngineType(AspectEngineDataFile dataFile) { if (dataFile.getEngine() != null) { return dataFile.getEngine().getClass().getSimpleName(); @@ -976,13 +1123,15 @@ private String getEngineType(AspectEngineDataFile dataFile) { *

    * A failure reading the download is thrown, for the caller to report. * A failure writing {@code tempFile} is logged here as the - * specification's write error. + * specification's write error. With MD5 verification on, a response + * with no Content-MD5 header is warned about and applied, and one whose + * hash does not match is the Errors table's integrity row. */ private AutoUpdateStatus downloadOkResponse( HttpURLConnection connection, AspectEngineDataFile dataFile, FileWrapper tempFile, - boolean automatic) throws IOException { + String loggedUrl) throws IOException { logger.debug("HTTP Status is OK"); boolean verifyMd5 = dataFile.getConfiguration().getVerifyMd5(); boolean decompress = dataFile.getConfiguration().getDecompressContent(); @@ -1016,13 +1165,21 @@ private AutoUpdateStatus downloadOkResponse( return AutoUpdateStatus.AUTO_UPDATE_ERR_READING_STREAM; } - if (verifyMd5 && !md5Matches(md, connection)) { - logger.warn("An error occurred during the integrity check of new " + - "data file for " + getEngineType(dataFile) + "." + - retryNote(automatic) + - " Error detail: the MD5 hash of the download does not " + - "match the Content-MD5 header"); - return AutoUpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED; + if (verifyMd5) { + String header = connection.getHeaderField("Content-MD5"); + String computed = DatatypeConverter.printHexBinary(md.digest()) + .toLowerCase(Locale.ROOT); + if (header == null) { + logger.warn("No MD5 hash included in data update response " + + "for '{}'. Unable to verify data integrity", loggedUrl); + } else if (header.equalsIgnoreCase(computed) == false) { + logCheckFailure("Integrity check failed. MD5 hash in HTTP " + + "response '" + header + "' for '" + + getEngineType(dataFile) + "'data update does not " + + "match calculated hash for the downloaded file '" + + computed + "'.", null); + return AutoUpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED; + } } return AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS; } @@ -1107,12 +1264,6 @@ private void writeBodyTo(FileWrapper tempFile, InputStream body) throws IOExcept } } - private static boolean md5Matches(MessageDigest md, HttpURLConnection connection) { - String header = connection.getHeaderField("Content-MD5"); - String computed = DatatypeConverter.printHexBinary(md.digest()); - return header != null && header.equalsIgnoreCase(computed); - } - @Override public void close() { if (closeFutureFactory) { @@ -1125,8 +1276,7 @@ public void close() { } private AutoUpdateStatus checkForUpdateFromUrl( - AspectEngineDataFile dataFile, - boolean automatic) { + AspectEngineDataFile dataFile) { if (dataFile.getDataFilePath() == null || dataFile.getDataFilePath().isEmpty()) { // There is no data file path specified so perform the // update entirely in memory. @@ -1134,7 +1284,7 @@ private AutoUpdateStatus checkForUpdateFromUrl( FileWrapper uncompressedData = new FileWrapperMemory(); AutoUpdateStatus memoryDownloadResult = - downloadFile(dataFile, uncompressedData, automatic); + downloadFile(dataFile, uncompressedData); if (memoryDownloadResult != AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS) { return memoryDownloadResult; } @@ -1162,10 +1312,10 @@ private AutoUpdateStatus checkForUpdateFromUrl( if (dataFile.getTempDataDirPath() == null || dataFile.getTempDataDirPath().isEmpty()) { - logger.error("The data file '" + dataFile.getIdentifier() + + logCheckFailure("The data file '" + dataFile.getIdentifier() + "' is checking for updates but does not have a temporary " + - "file path configured."); - return AutoUpdateStatus.AUTO_UPDATE_REFRESH_FAILED; + "file path configured.", null); + return AutoUpdateStatus.AUTO_UPDATE_TEMP_PATH_NOT_SET; } Path uncompressedTempFile = Paths.get( dataFile.getTempDataDirPath(), @@ -1174,7 +1324,7 @@ private AutoUpdateStatus checkForUpdateFromUrl( try { // Check if there is an update and download it if there is AutoUpdateStatus result = - downloadFile(dataFile, uncompressedData, automatic); + downloadFile(dataFile, uncompressedData); if (result == AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS) { WatchKey watchKey = dataFile.getConfiguration().getWatchKey(); @@ -1223,8 +1373,8 @@ private AutoUpdateStatus checkForUpdateFromUrl( } return result; } catch (Throwable e) { - logger.error("Exception thrown", e); - return AutoUpdateStatus.AUTO_UPDATE_REFRESH_FAILED; + logUnhandledError(dataFile, e); + return AutoUpdateStatus.AUTO_UPDATE_UNKNOWN_ERROR; } finally { // Make sure the temp files are cleaned up try { diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/HostLookup.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/HostLookup.java new file mode 100644 index 000000000..fc4939cad --- /dev/null +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/HostLookup.java @@ -0,0 +1,207 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.InetAddress; +import java.net.Proxy; +import java.net.ProxySelector; +import java.net.SocketTimeoutException; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Looks up the host of a URL within a time limit. The connect timeout of an + * {@link java.net.HttpURLConnection} does not cover the name lookup, which + * waits as long as the system resolver does. Looking the host up here first + * leaves the address in the JVM's cache for the connection to use. Callers + * waiting on the same host share one lookup, so a resolver that does not + * answer holds one thread, not one for each request. + *

    + * Two limits follow from relying on that cache. With the security property + * {@code networkaddress.cache.ttl} set to 0 nothing is cached, so the + * connection looks the host up again itself, with no limit. And every + * request is looked up here, including one the connection would send on a + * kept-alive socket with no lookup at all, so once the cached address + * expires, 30 seconds after it was found by default, a resolver that does + * not answer fails that request although the open socket could have served + * it. + */ +final class HostLookup { + + /** + * Message of the exception thrown when a lookup takes longer than the + * timeout. The arguments are the host and the timeout in milliseconds. + */ + static final String MessageLookupTimedOut = + "Looking up the host '%s' took longer than the timeout of %d ms."; + + /** + * Finds the addresses of a host. + */ + interface Resolver { + InetAddress[] resolve(String host) throws UnknownHostException; + } + + /** + * The lookup {@link HttpClientDefault} uses, through the JVM's resolver. + */ + static final HostLookup DEFAULT = new HostLookup(InetAddress::getAllByName); + + private static final Executor LOOKUPS = Executors.newCachedThreadPool( + runnable -> { + Thread thread = new Thread(runnable, "51Degrees host lookup"); + thread.setDaemon(true); + return thread; + }); + + private final Resolver resolver; + + private final ConcurrentMap> + inProgress = new ConcurrentHashMap<>(); + + HostLookup(Resolver resolver) { + this.resolver = resolver; + } + + /** + * Look up the host of the URL, waiting at most the timeout. Nothing is + * looked up when the timeout is 0, which means no limit, or when a proxy + * makes the request, because the proxy looks up the host. + * @param url the URL to be requested + * @param timeoutMillis the longest wait in milliseconds + * @throws SocketTimeoutException if the lookup takes longer + * @throws UnknownHostException if the host has no address + * @throws IOException if the lookup fails in another way + */ + void lookUp(URL url, int timeoutMillis) throws IOException { + String host = url.getHost(); + if (timeoutMillis <= 0 || + host == null || + host.isEmpty() || + usesProxy(url)) { + return; + } + String key = host.toLowerCase(Locale.ROOT); + CompletableFuture started = new CompletableFuture<>(); + CompletableFuture lookup = + inProgress.putIfAbsent(key, started); + if (lookup == null) { + lookup = started; + start(key, host, started); + } + try { + lookup.get(timeoutMillis, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + throw new SocketTimeoutException(String.format( + MessageLookupTimedOut, host, timeoutMillis)); + } catch (ExecutionException e) { + throw failure(host, e.getCause()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + InterruptedIOException interrupted = new InterruptedIOException( + "Interrupted while looking up the host '" + host + "'."); + interrupted.initCause(e); + throw interrupted; + } + } + + /** + * Start the lookup on its own thread. A finished lookup is removed from + * those in progress, so the next request looks up again, which the JVM's + * cache answers while the address is cached. + */ + private void start( + String key, + String host, + CompletableFuture lookup) { + try { + LOOKUPS.execute(() -> { + try { + lookup.complete(resolver.resolve(host)); + } catch (Throwable e) { + lookup.completeExceptionally(e); + } finally { + inProgress.remove(key, lookup); + } + }); + } catch (RuntimeException | Error e) { + inProgress.remove(key, lookup); + lookup.completeExceptionally(e); + throw e; + } + } + + /** + * The exception for a lookup that failed, thrown on the caller's thread + * with the lookup's own exception as the cause. + */ + private static IOException failure(String host, Throwable cause) { + if (cause instanceof UnknownHostException) { + UnknownHostException unknown = + new UnknownHostException(cause.getMessage()); + unknown.initCause(cause); + return unknown; + } + return new IOException( + "Looking up the host '" + host + "' failed.", cause); + } + + /** + * Whether the default proxy selector sends the request through a proxy. + * A URL it cannot read counts as one, so it is not looked up here. + */ + private static boolean usesProxy(URL url) { + ProxySelector selector = ProxySelector.getDefault(); + if (selector == null) { + return false; + } + List proxies; + try { + proxies = selector.select(url.toURI()); + } catch (URISyntaxException | IllegalArgumentException e) { + return true; + } + if (proxies == null) { + return false; + } + for (Proxy proxy : proxies) { + if (proxy.type() != Proxy.Type.DIRECT) { + return true; + } + } + return false; + } +} diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/HttpClientDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/HttpClientDefault.java index 3dfaa5282..4759da850 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/HttpClientDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/HttpClientDefault.java @@ -29,8 +29,26 @@ import java.net.URL; import java.util.Map; +/** + * Makes requests with {@link HttpURLConnection}. The connect timeout set on + * a connection also limits the lookup of its host, which the connection's + * own timeouts do not cover. A host is not looked up when the default + * {@link java.net.ProxySelector} sends its request through a proxy. A + * subclass whose {@link #connect(URL)} opens the connection through a proxy + * of its own still has the host looked up here, so its requests fail when + * the host cannot be found locally although the proxy could reach it. + */ public class HttpClientDefault implements HttpClient { + private final HostLookup hostLookup; + + public HttpClientDefault() { + this(HostLookup.DEFAULT); + } + + HttpClientDefault(HostLookup hostLookup) { + this.hostLookup = hostLookup; + } @Override public HttpURLConnection connect(URL url) throws IOException { @@ -49,6 +67,7 @@ public String postData( connection.setRequestProperty(e.getKey(), e.getValue()); } } + lookUpHost(connection); if (data == null) { connection.setDoOutput(false); } else { @@ -86,6 +105,7 @@ public String getResponseString( connection.setRequestProperty(e.getKey(), e.getValue()); } } + lookUpHost(connection); BufferedReader in = new BufferedReader( new InputStreamReader(connection.getResponseCode() / 100 == 2 ? @@ -100,4 +120,12 @@ public String getResponseString( in.close(); return response.toString(); } + + /** + * Look up the connection's host within its connect timeout, before the + * connection makes its own lookup, which has no limit. + */ + private void lookUpHost(HttpURLConnection connection) throws IOException { + hostLookup.lookUp(connection.getURL(), connection.getConnectTimeout()); + } } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyReason.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyReason.java index e9c01f3c1..60697b59e 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyReason.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyReason.java @@ -33,5 +33,10 @@ public enum MissingPropertyReason { PropertyExcludedFromEngineConfiguration, PropertyNotAccessibleWithResourceKey, ProductNotAccessibleWithResourceKey, + /** + * A request to the cloud service failed, so no data was available to + * populate the property. + */ + CloudRequestFailed, Unknown } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyService.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyService.java index af24e0696..359f4c872 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyService.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyService.java @@ -29,6 +29,8 @@ import java.util.List; +import static fiftyone.pipeline.engines.Constants.MissingPropertyMessages; + /** * Service used by {@link AspectEngine}s to report the reason for a property * not being present in the results. @@ -59,4 +61,29 @@ MissingPropertyResult getMissingPropertyReason( MissingPropertyResult getMissingPropertyReason( String propertyName, List> engines); + + /** + * Get the reason for the property not being found when the cloud + * request for the Flow Data failed, which is CloudRequestFailed. The + * message names the Data Key of the first engine, or Unknown when there + * is none. + * + * @param propertyName name of the {@link AspectPropertyMetaData} + * @param engines that populate the Element Data + * @return reason the property was not found + */ + default MissingPropertyResult getCloudRequestFailedReason( + String propertyName, + List> engines) { + String dataKey = engines == null || engines.isEmpty() ? + MissingPropertyMessages.UNKNOWN_DATA_KEY : + engines.get(0).getElementDataKey(); + return new MissingPropertyResult( + MissingPropertyReason.CloudRequestFailed, + String.format( + MissingPropertyMessages.PREFIX, + propertyName, + dataKey) + + MissingPropertyMessages.CLOUD_REQUEST_FAILED); + } } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyServiceDefault.java index 69f94920e..1c9206075 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/MissingPropertyServiceDefault.java @@ -23,11 +23,15 @@ package fiftyone.pipeline.engines.services; import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.typed.TypedKey; import fiftyone.pipeline.engines.data.AspectData; import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.engines.flowelements.AspectEngine; import fiftyone.pipeline.engines.flowelements.CloudAspectEngine; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import java.lang.reflect.Method; import java.util.List; import java.util.stream.Collectors; @@ -46,6 +50,11 @@ public class MissingPropertyServiceDefault implements MissingPropertyService { private static final Object lock = new Object(); + /** + * The start of the name of a typed accessor for a property. + */ + private static final String ACCESSOR_PREFIX = "get"; + /** * Private constructor to prevent new copies */ @@ -76,48 +85,56 @@ public MissingPropertyResult getMissingPropertyReason( MissingPropertyReason reason = MissingPropertyReason.Unknown; - AspectPropertyMetaData property = null; + boolean cloud = engine instanceof CloudAspectEngine; + boolean onPremise = engine instanceof OnPremiseAspectEngine; + // An On-premise Engine loads its metadata when it is built. + boolean loaded = cloud == false || engine.hasLoadedProperties(); - // We know the property has extends AspectPropertyMetaData as it - // is a constraint of the AspectEngine interface, so don't check this - // cast - for (Object currentPropertyObject : engine.getProperties()) { - AspectPropertyMetaData currentProperty = - (AspectPropertyMetaData)currentPropertyObject; - if (currentProperty.getName().equalsIgnoreCase(propertyName)) { - property = currentProperty; - break; - } - } - - if (property != null) { - // Check if the property is available in the data file that is - // being used by the engine. - boolean containsDataTier = false; - for (String tier : property.getDataTiersWherePresent()) { - if (tier.equals(engine.getDataSourceTier())) { - containsDataTier = true; + AspectPropertyMetaData property = null; + if (loaded) { + // We know the property has extends AspectPropertyMetaData as it + // is a constraint of the AspectEngine interface, so don't check + // this cast + for (Object currentPropertyObject : engine.getProperties()) { + AspectPropertyMetaData currentProperty = + (AspectPropertyMetaData)currentPropertyObject; + if (currentProperty.getName().equalsIgnoreCase(propertyName)) { + property = currentProperty; break; } } + } - if (containsDataTier == false) { + // The metadata lists the property. Only an On-premise Engine has a + // data file tier to compare. + if (property != null) { + if (onPremise && hasDataTier(property, engine) == false) { reason = MissingPropertyReason.DataFileUpgradeRequired; } - // Check if the property is excluded from the results. else if (property.isAvailable() == false) { reason = MissingPropertyReason.PropertyExcludedFromEngineConfiguration; } } - else { - if (CloudAspectEngine.class.isAssignableFrom(engine.getClass())) { - if (engine.getProperties().size() == 0) { - reason = MissingPropertyReason.ProductNotAccessibleWithResourceKey; - } - else { - reason = MissingPropertyReason.PropertyNotAccessibleWithResourceKey; - } + + if (reason == MissingPropertyReason.Unknown && cloud) { + if (loaded) { + reason = engine.getProperties().isEmpty() ? + MissingPropertyReason.ProductNotAccessibleWithResourceKey : + MissingPropertyReason.PropertyNotAccessibleWithResourceKey; } + else if (((CloudAspectEngine) engine) + .propertyListRequestFailed()) { + reason = MissingPropertyReason.CloudRequestFailed; + } + } + + // The metadata does not list the property, but the Element Data has + // a typed accessor for it, so another data file has it. + if (reason == MissingPropertyReason.Unknown && + onPremise && + property == null && + hasTypedAccessor(engine, propertyName)) { + reason = MissingPropertyReason.DataFileUpgradeRequired; } // Build the message string to return to the caller. @@ -128,12 +145,20 @@ else if (property.isAvailable() == false) { engine.getElementDataKey())); switch (reason) { case DataFileUpgradeRequired: - message.append(String.format( - MissingPropertyMessages.DATA_UPGRADE_REQUIRED, - stringJoin( - property.getDataTiersWherePresent(), - ","), - engine.getClass().getSimpleName())); + List tiers = property == null ? + null : + property.getDataTiersWherePresent(); + if (tiers == null || tiers.isEmpty()) { + message.append( + MissingPropertyMessages.DATA_UPGRADE_REQUIRED_TIERS_NOT_KNOWN); + } else { + message.append(String.format( + MissingPropertyMessages.DATA_UPGRADE_REQUIRED, + stringJoin( + tiers, + MissingPropertyMessages.LIST_SEPARATOR), + engine.getClass().getSimpleName())); + } break; case PropertyExcludedFromEngineConfiguration: message.append(MissingPropertyMessages.PROPERTY_EXCLUDED); @@ -151,7 +176,12 @@ else if (property.isAvailable() == false) { message.append(String.format( MissingPropertyMessages.PROPERTY_NOT_IN_CLOUD_RESOURCE, engine.getElementDataKey(), - stringJoin(available, ", "))); + stringJoin( + available, + MissingPropertyMessages.LIST_SEPARATOR))); + break; + case CloudRequestFailed: + message.append(MissingPropertyMessages.CLOUD_REQUEST_FAILED); break; case Unknown: message.append(MissingPropertyMessages.UNKNOWN); @@ -163,6 +193,43 @@ else if (property.isAvailable() == false) { return new MissingPropertyResult(reason, message.toString()); } + /** + * Whether the engine's data file tier is among the tiers the property + * is present in. + */ + private static boolean hasDataTier( + AspectPropertyMetaData property, + AspectEngine engine) { + List tiers = property.getDataTiersWherePresent(); + return tiers != null && tiers.contains(engine.getDataSourceTier()); + } + + /** + * Whether the type of the engine's Element Data, counting the types it + * inherits, has a typed accessor for the property, being a public + * method with no parameters named 'get' and the property name, compared + * ignoring case, that returns an {@link AspectPropertyValue}. + */ + private static boolean hasTypedAccessor( + AspectEngine engine, + String propertyName) { + TypedKey key = engine.getTypedDataKey(); + Class type = key == null ? null : key.getType(); + if (type == null) { + return false; + } + String accessor = ACCESSOR_PREFIX + propertyName; + for (Method method : type.getMethods()) { + if (method.getParameterCount() == 0 && + method.getName().equalsIgnoreCase(accessor) && + AspectPropertyValue.class.isAssignableFrom( + method.getReturnType())) { + return true; + } + } + return false; + } + @Override public MissingPropertyResult getMissingPropertyReason( String propertyName, diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/AspectDataPoolTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/AspectDataPoolTests.java new file mode 100644 index 000000000..b71f132f0 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/AspectDataPoolTests.java @@ -0,0 +1,208 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.flowelements; + +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectDataBase; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.data.AspectPropertyMetaDataDefault; +import fiftyone.pipeline.engines.data.ProcessCallable; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyReason; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; + +/** + * Aspect Data that opts in to pooling starts each request with nothing of + * the request it served before. Besides its values, that is the engines that + * shared it, the process tasks added to it and a failed cloud request. + * @see Specification + */ +public class AspectDataPoolTests { + + private static final Logger LOGGER = + LoggerFactory.getLogger(AspectDataPoolTests.class); + + private static final String FAIL_EVIDENCE = "query.fail"; + + public static class PooledAspectData extends AspectDataBase { + public PooledAspectData( + Logger logger, + FlowData flowData, + AspectEngine engine) { + super( + logger, + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + } + + @Override + protected boolean isPoolable() { + return true; + } + } + + /** + * Writes valueone. Given the fail Evidence, it instead leaves everything + * a request can leave in Aspect Data besides values: a second engine + * sharing it, a process task that failed and a failed cloud request. + */ + private static class PoolEngine + extends AspectEngineBase { + private final List properties; + private final PoolEngine partner; + + PoolEngine(PoolEngine partner) { + super(LOGGER, (flowData, element) -> new PooledAspectData( + LOGGER, + flowData, + (AspectEngine) element)); + this.partner = partner; + properties = Arrays.asList( + (AspectPropertyMetaData) new AspectPropertyMetaDataDefault( + "valueone", this, "", int.class, + new ArrayList(), true), + (AspectPropertyMetaData) new AspectPropertyMetaDataDefault( + "valuetwo", this, "", int.class, + new ArrayList(), true)); + } + + @Override + protected void processEngine( + FlowData data, + PooledAspectData aspectData) { + if (data.getEvidence().get(FAIL_EVIDENCE) != null) { + aspectData.addEngine(partner); + aspectData.addProcessCallable(new ProcessCallable(this) { + @Override + public Void call() throws Exception { + throw new Exception("The task failed."); + } + }); + aspectData.setCloudRequestFailed(); + } else { + aspectData.put("valueone", 1); + } + } + + @Override + public List getProperties() { + return properties; + } + + @Override + public String getDataSourceTier() { + return "test"; + } + + @Override + public String getElementDataKey() { + return "pooled-aspect"; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist( + Collections.singletonList(FAIL_EVIDENCE)); + } + + @Override + protected void unmanagedResourcesCleanup() { + } + } + + private static PooledAspectData process( + Pipeline pipeline, + PoolEngine engine, + boolean fail) throws Exception { + try (FlowData data = pipeline.createFlowData()) { + if (fail) { + data.addEvidence(FAIL_EVIDENCE, "yes"); + } + data.process(); + return data.getFromElement(engine); + } + } + + @Test + public void reusedAspectData_KeepsNothingOfTheRequestBefore() + throws Exception { + PoolEngine partner = new PoolEngine(null); + PoolEngine engine = new PoolEngine(partner); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElement(engine) + .setAutoCloseElements(true) + .build()) { + PooledAspectData failed = process(pipeline, engine, true); + + try (FlowData data = pipeline.createFlowData()) { + data.process(); + PooledAspectData reused = data.getFromElement(engine); + assertSame( + "The instance was not reused, so this proves nothing", + failed, + reused); + assertEquals( + "It kept the engines of the request before", + Collections.singletonList(engine), + reused.getEngines()); + assertEquals( + "It kept the failed task of the request before", + 1, + reused.get("valueone")); + try { + reused.get("valuetwo"); + fail("valuetwo was never written"); + } catch (PropertyMissingException e) { + assertNotEquals( + "It kept the failed cloud request of the request " + + "before", + MissingPropertyReason.CloudRequestFailed, + e.getReason()); + } + } + } finally { + partner.close(); + } + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBasePropertiesTest.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBasePropertiesTest.java new file mode 100644 index 000000000..6bdef8e56 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/AspectEngineBuilderBasePropertiesTest.java @@ -0,0 +1,89 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.flowelements; + +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngineBuilder; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; + +/** + * A list of properties given as one string, as a configuration file gives + * it, is a comma-separated list. + */ +public class AspectEngineBuilderBasePropertiesTest { + + private static EmptyEngineBuilder builder() { + return new EmptyEngineBuilder(LoggerFactory.getILoggerFactory()); + } + + @Test + public void setProperties_CommasWithoutSpaces_EachProperty() { + EmptyEngineBuilder builder = builder() + .setProperties("IsMobile,BrowserName,PlatformName"); + assertEquals( + Arrays.asList("IsMobile", "BrowserName", "PlatformName"), + builder.getProperties()); + } + + @Test + public void setProperties_SpacesAroundNames_Trimmed() { + EmptyEngineBuilder builder = builder() + .setProperties(" IsMobile, BrowserName ,PlatformName , DeviceType"); + assertEquals( + Arrays.asList("IsMobile", "BrowserName", "PlatformName", + "DeviceType"), + builder.getProperties()); + } + + @Test + public void setProperties_EmptyEntries_LeftOut() { + EmptyEngineBuilder builder = builder() + .setProperties("IsMobile,,BrowserName,"); + assertEquals( + Arrays.asList("IsMobile", "BrowserName"), + builder.getProperties()); + } + + @Test + public void setProperties_OneName_OneProperty() { + EmptyEngineBuilder builder = builder().setProperties("IsMobile"); + assertEquals( + Collections.singletonList("IsMobile"), + builder.getProperties()); + } + + @Test + public void setProperties_Repeated_AddedOnce() { + EmptyEngineBuilder builder = builder() + .setProperties("IsMobile,IsMobile") + .setProperties("BrowserName, IsMobile"); + assertEquals( + Arrays.asList("IsMobile", "BrowserName"), + builder.getProperties()); + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/AspectEngineLazyLoadTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/AspectEngineLazyLoadTests.java index fd5b44126..45fc839d0 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/AspectEngineLazyLoadTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/AspectEngineLazyLoadTests.java @@ -29,12 +29,10 @@ import fiftyone.pipeline.core.typed.TypedKey; import fiftyone.pipeline.engines.configuration.ExecutorServiceFactory; import fiftyone.pipeline.engines.configuration.LazyLoadingConfiguration; -import fiftyone.pipeline.engines.exceptions.LazyLoadTimeoutException; import fiftyone.pipeline.engines.testhelpers.data.EmptyEngineData; import fiftyone.pipeline.engines.testhelpers.data.MockFlowData; import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngine; import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngineBuilder; -import fiftyone.pipeline.exceptions.AggregateException; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -45,7 +43,6 @@ import java.util.HashMap; import java.util.Map; -import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -121,21 +118,17 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { // Process the data. engine.process(mockData); - // Check that the task is not complete when process returns. - boolean processReturnedBeforeTaskComplete = - engineData[0].getProcessFuture().isDone() == false; + // Lazy loading has no effect, so processing has finished when + // process returns. + boolean processReturnedAfterTaskComplete = + engineData[0].getProcessFuture().isDone(); // Assert assertEquals(1, engineData[0].getValueOne()); - // Check that the task is now complete (because the code to get - // the property value will wait until it is complete) - boolean valueReturnedAfterTaskComplete = - engineData[0].getProcessFuture().isDone() == true; - assertTrue(processReturnedBeforeTaskComplete); - assertTrue(valueReturnedAfterTaskComplete); + assertTrue(processReturnedAfterTaskComplete); } - @Test(expected = LazyLoadTimeoutException.class) + @Test public void AspectEngineLazyLoad_PropertyTimeout() throws Exception { // Arrange // Set the process time to double the configured timeout @@ -163,15 +156,14 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { // Process the data engine.process(mockData); - // Attempt to get the value. This should cause the timeout - // to be triggered. + // Lazy loading has no effect, so nothing waits and the value is + // there, however long processing took. int result = engineData[0].getValueOne(); - // No asserts needed. Just the ExpectedException attribute - // on the method. + assertEquals(1, result); } - @Test(expected = CancellationException.class) + @Test public void AspectEngineLazyLoad_ProcessCancelled() throws Exception { // Arrange // Set the process time to half the configured timeout @@ -198,12 +190,13 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { // Process the data engine.process(mockData); - // Trigger a cancellation. + // Processing has finished, so there is nothing to cancel. mockData.stop(); engineData[0].getProcessFuture().cancel(true); // Attempt to get the value. int result = engineData[0].getValueOne(); + assertEquals(1, result); } @Test @@ -231,18 +224,17 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { any(TypedKey.class), any(FlowElement.DataFactory.class)); - // Process the data - engine.process(mockData); - - // Attempt to get the value. + // Process the data. Lazy loading has no effect, so the Engine's + // exception comes from process. + Exception thrown = null; try { - int result = engineData[0].getValueOne(); + engine.process(mockData); } catch (Exception e) { - assertTrue(e instanceof RuntimeException); - assertTrue(e.getCause() != null); - assertEquals("java.lang.Exception: " + exceptionMessage, e.getCause().getMessage()); + thrown = e; } + assertTrue(thrown != null); + assertEquals(exceptionMessage, thrown.getMessage()); } @Test @@ -283,19 +275,18 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { any(TypedKey.class), any(FlowElement.DataFactory.class)); - // Process the data twice - engine.process(mockData); - engine2.process(mockData); - - // Attempt to get the value. - try { - int result = engineData[0].getValueOne(); - } - catch (Exception e) { - assertTrue(e instanceof AggregateException); - assertEquals(2, e.getSuppressed().length); - assertEquals("java.lang.Exception: " + exceptionMessage, e.getSuppressed()[0].getMessage()); - assertEquals("java.lang.Exception: " + exceptionMessage, e.getSuppressed()[0].getMessage()); + // Process the data twice. Lazy loading has no effect, so each + // Engine's exception comes from its own process call. + int thrown = 0; + for (EmptyEngine each : new EmptyEngine[] {engine, engine2}) { + try { + each.process(mockData); + } + catch (Exception e) { + assertEquals(exceptionMessage, e.getMessage()); + thrown++; + } } + assertEquals(2, thrown); } } diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/BuilderDefaultValueTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/BuilderDefaultValueTests.java new file mode 100644 index 000000000..b29c14985 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/BuilderDefaultValueTests.java @@ -0,0 +1,73 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.flowelements; + +import fiftyone.pipeline.annotations.DefaultValue; +import fiftyone.pipeline.engines.configuration.DataFileConfigurationBuilderBase; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Every configuration setter of the engine builders gives its default + * value beside it, with {@link DefaultValue}. See + * reference-implementation-notes.md line 79: + * "In all cases, there SHOULD be some mechanism to specify the default + * values alongside configuration property setters." + */ +public class BuilderDefaultValueTests { + + static List missingDefaults(Class... builders) { + List missing = new ArrayList<>(); + for (Class builder : builders) { + for (Method method : builder.getDeclaredMethods()) { + if (Modifier.isPublic(method.getModifiers()) && + method.getName().startsWith("set") && + method.isSynthetic() == false && + method.isBridge() == false && + method.getAnnotation(DefaultValue.class) == null) { + missing.add(builder.getSimpleName() + "#" + + method.getName() + + Arrays.toString(method.getParameterTypes())); + } + } + } + return missing; + } + + @Test + public void engineBuilderSetters_GiveTheirDefaults() { + assertEquals(new ArrayList(), missingDefaults( + AspectEngineBuilderBase.class, + OnPremiseAspectEngineBuilderBase.class, + SingleFileAspectEngineBuilderBase.class, + CloudPipelineBuilderBase.class, + DataFileConfigurationBuilderBase.class)); + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/ConfigurationValueTypesTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/ConfigurationValueTypesTests.java new file mode 100644 index 000000000..495df35a8 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/ConfigurationValueTypesTests.java @@ -0,0 +1,89 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.flowelements; + +import fiftyone.pipeline.core.configuration.ElementOptions; +import fiftyone.pipeline.core.configuration.PipelineOptions; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.testhelpers.flowelements.ConfiguredEngineBuilder; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; + +/** + * Configuration files hold a list of strings as a comma-separated string and + * an enumeration value as the name of the value, the same in every language. + *

    + * Specification: pipeline-specification/features/pipeline-configuration.md + * line 50 at 51Degrees/specifications 706de1f0, "Configuration files MUST be + * human-readable and consistent between languages.", and lines 60 and 61, + * "list of strings (represented as a comma-separated string)" and + * "enumeration value (represented as the string name of the value)". + */ +public class ConfigurationValueTypesTests { + + private static void build(String name, String value) throws Exception { + ElementOptions element = new ElementOptions(); + element.builderName = "ConfiguredEngine"; + element.buildParameters.put(name, value); + PipelineOptions options = new PipelineOptions(); + options.elements.add(element); + try (Pipeline ignored = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .buildFromConfiguration(options)) { + // Building is enough. + } + } + + @Test + public void properties_CommaSeparated_AreEachProperty() throws Exception { + build("Properties", "IsMobile,BrowserName"); + + assertEquals( + Arrays.asList("IsMobile", "BrowserName"), + ConfiguredEngineBuilder.lastProperties); + } + + @Test + public void properties_CommaAndSpaceSeparated_AreEachProperty() + throws Exception { + build("Properties", "IsMobile, BrowserName ,PlatformName"); + + assertEquals( + Arrays.asList("IsMobile", "BrowserName", "PlatformName"), + ConfiguredEngineBuilder.lastProperties); + } + + @Test + public void enumeration_ByName_IsSet() throws Exception { + build("Speed", "FAST"); + + assertEquals( + ConfiguredEngineBuilder.Speed.FAST, + ConfiguredEngineBuilder.lastSpeed); + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/LazyLoadingHasNoEffectTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/LazyLoadingHasNoEffectTests.java new file mode 100644 index 000000000..0a0c9c095 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/LazyLoadingHasNoEffectTests.java @@ -0,0 +1,168 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.configuration.ExecutorServiceFactory; +import fiftyone.pipeline.engines.configuration.LazyLoadingConfiguration; +import fiftyone.pipeline.engines.testhelpers.data.EmptyEngineData; +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngine; +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngineBuilder; +import org.junit.After; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * The deprecated lazy loading option has no effect. An Engine with it set + * processes on the thread that processes the Flow Data, uses no executor, + * and gives the same values as one without it, whatever the timeout. + * @see Specification + */ +public class LazyLoadingHasNoEffectTests { + + private static final long PROCESS_COST_MILLIS = 200; + + /** + * How much sooner than the process cost process may seem to return. On + * Windows, Thread.sleep can wake a millisecond or two early by the + * System.nanoTime clock. Processing in the background would return + * almost at once, far below this. + */ + private static final long SLEEP_EARLY_TOLERANCE_MILLIS = 20; + + private static final int LONG_TIMEOUT_MILLIS = 5000; + + private static final int SHORT_TIMEOUT_MILLIS = 10; + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + private final AtomicInteger executorsCreated = new AtomicInteger(); + + private final AtomicInteger threadsStarted = new AtomicInteger(); + + private ExecutorService executor; + + @After + public void cleanup() { + if (executor != null) { + executor.shutdownNow(); + } + } + + private ExecutorServiceFactory countingFactory() { + return new ExecutorServiceFactory() { + @Override + public ExecutorService create() { + executorsCreated.incrementAndGet(); + executor = Executors.newSingleThreadExecutor(runnable -> { + threadsStarted.incrementAndGet(); + return new Thread(runnable); + }); + return executor; + } + }; + } + + private void processAndCheck(EmptyEngine engine) throws Exception { + engine.setProcessCost(PROCESS_COST_MILLIS); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("test.value", "1"); + long start = System.nanoTime(); + data.process(); + long elapsedMillis = (System.nanoTime() - start) / 1_000_000; + + EmptyEngineData result = data.get(EmptyEngineData.class); + assertTrue( + "Processing should be finished when process returns", + result.getProcessFuture().isDone()); + assertTrue( + "process should return only once the Engine has processed, " + + "but returned after " + elapsedMillis + " ms", + elapsedMillis >= + PROCESS_COST_MILLIS - SLEEP_EARLY_TOLERANCE_MILLIS); + assertEquals(1, result.getValueOne()); + assertEquals(2, result.getValueTwo()); + } + assertEquals( + "No thread should be started for lazy loading", + 0, + threadsStarted.get()); + } + + @Test + public void lazyLoading_NotSet_ProcessesOnCallingThread() + throws Exception { + processAndCheck(new EmptyEngineBuilder(loggerFactory).build()); + } + + @Test + public void lazyLoading_LongTimeout_ProcessesOnCallingThread() + throws Exception { + processAndCheck(new EmptyEngineBuilder(loggerFactory) + .setLazyLoading(new LazyLoadingConfiguration( + LONG_TIMEOUT_MILLIS, + countingFactory())) + .build()); + } + + @Test + public void lazyLoading_TimeoutShorterThanProcessing_GivesValues() + throws Exception { + processAndCheck(new EmptyEngineBuilder(loggerFactory) + .setLazyLoading(new LazyLoadingConfiguration( + SHORT_TIMEOUT_MILLIS, + countingFactory())) + .build()); + } + + @Test + public void lazyLoading_Set_CreatesNoExecutor() throws Exception { + EmptyEngine engine = new EmptyEngineBuilder(loggerFactory) + .setLazyLoading(new LazyLoadingConfiguration( + LONG_TIMEOUT_MILLIS, + countingFactory())) + .build(); + try { + assertEquals( + "Setting lazy loading should create no executor", + 0, + executorsCreated.get()); + } finally { + engine.close(); + } + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/PrePackagedPipelineBuilderBaseTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/PrePackagedPipelineBuilderBaseTests.java new file mode 100644 index 000000000..5634edef0 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/PrePackagedPipelineBuilderBaseTests.java @@ -0,0 +1,78 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.flowelements; + +import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.configuration.CacheConfiguration; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * The defaults of a pre-packaged Pipeline builder come from {@link Constants} + * and keep the values they have always had. + */ +public class PrePackagedPipelineBuilderBaseTests { + + private static class TestBuilder + extends PrePackagedPipelineBuilderBase { + } + + @Test + public void Defaults_FromConstants() { + TestBuilder builder = new TestBuilder(); + assertEquals(Constants.DEFAULT_LAZY_LOADING, builder.lazyLoading); + assertEquals( + Constants.DEFAULT_LAZY_LOADING_TIMEOUT_MILLIS, + builder.lazyLoadingTimeoutMillis); + assertEquals(Constants.DEFAULT_RESULTS_CACHE, builder.resultsCache); + assertEquals( + Constants.DEFAULT_RESULTS_CACHE_SIZE, + builder.resultsCacheSize); + } + + @Test + public void Defaults_Values() { + assertFalse(Constants.DEFAULT_LAZY_LOADING); + assertEquals(1000, Constants.DEFAULT_LAZY_LOADING_TIMEOUT_MILLIS); + assertFalse(Constants.DEFAULT_RESULTS_CACHE); + assertEquals(1000, Constants.DEFAULT_RESULTS_CACHE_SIZE); + assertEquals( + CacheConfiguration.defaultSize, + Constants.DEFAULT_RESULTS_CACHE_SIZE); + } + + @Test + public void UseWithoutSize_KeepsDefaults() { + TestBuilder builder = new TestBuilder() + .useLazyLoading() + .useResultsCache(); + assertEquals( + Constants.DEFAULT_LAZY_LOADING_TIMEOUT_MILLIS, + builder.lazyLoadingTimeoutMillis); + assertEquals( + Constants.DEFAULT_RESULTS_CACHE_SIZE, + builder.resultsCacheSize); + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/RelativePathTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/RelativePathTests.java new file mode 100644 index 000000000..ddfa84efd --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/RelativePathTests.java @@ -0,0 +1,204 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.flowelements; + +import fiftyone.common.wrappers.io.FileWrapperFactoryDefault; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.DataUpdateServiceDefault; +import fiftyone.pipeline.engines.services.HttpClient; +import fiftyone.pipeline.engines.services.update.FutureFactory; +import fiftyone.pipeline.engines.testhelpers.data.EmptyEngineData; +import fiftyone.pipeline.engines.testhelpers.flowelements.TextFileEngine; +import fiftyone.pipeline.engines.testhelpers.flowelements.TextFileEngineBuilder; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.UUID; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * A data file path, and a temporary folder path, may be relative to the + * working directory of the process. + *

    + * Specification: pipeline-specification/features/file-system-paths.md lines + * 3 to 5 at 51Degrees/specifications 706de1f0, "Where file system paths are + * supplied as parameters, the Pipeline API MUST be able to work with relative + * paths. A relative path is resolved against the working directory of the + * process." + */ +public class RelativePathTests { + + /** + * The longest a test waits for the file system watcher to see a change. + * On macOS, Java's watch service polls, by default every 10 seconds, so + * an event can take 10 seconds or more to arrive. With a 10 second wait + * this test failed on both macOS jobs of java #190's release build + * (run 44702767584) at 10.21 s and 10.13 s. 30 seconds is three polls. + * The loop ends as soon as the change is seen, so a fast machine waits + * no longer than before. + */ + private static final long WAIT_FOR_EVENT_MILLIS = 30000; + + private static final long RETRY_MILLIS = 100; + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + /** + * The folder for the test's files, relative to the working directory. + */ + private Path folder; + + /** + * The data file, relative to the working directory. + */ + private String dataFile; + + @Before + public void createDataFile() throws Exception { + folder = Paths.get("target", "relative-" + UUID.randomUUID()); + Files.createDirectories(folder); + dataFile = folder.resolve("data.txt").toString(); + Files.write( + Paths.get(dataFile), + "Old".getBytes(StandardCharsets.UTF_8)); + assertFalse( + "The path in the test is relative", + Paths.get(dataFile).isAbsolute()); + } + + @After + public void deleteFiles() { + File[] files = folder.toFile().listFiles(); + if (files != null) { + for (File file : files) { + if (file.delete() == false) { + file.deleteOnExit(); + } + } + } + if (folder.toFile().delete() == false) { + folder.toFile().deleteOnExit(); + } + } + + @Test + public void dataFile_Relative_EngineReadsIt() throws Exception { + try (TextFileEngine engine = + new TextFileEngineBuilder(loggerFactory, null) + .setAutoUpdate(false) + .build(dataFile, false); + Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + assertEquals("Old", engine.getText()); + assertEquals(dataFile, engine.getDataFileMetaData().getDataFilePath()); + + data.process(); + assertEquals(3, data.get(EmptyEngineData.class).getValueOne()); + } + } + + @Test + public void tempDir_Relative_ResolvedAgainstTheWorkingDirectory() + throws Exception { + String tempDir = folder.resolve("temp").toString(); + try (TextFileEngine engine = + new TextFileEngineBuilder(loggerFactory, null) + .setAutoUpdate(false) + .setTempDirPath(tempDir) + .build(dataFile, false)) { + assertEquals( + Paths.get(tempDir).toAbsolutePath().normalize(), + Paths.get(engine.getTempDataDirPath()) + .toAbsolutePath().normalize()); + assertEquals( + Paths.get(tempDir, "data.txt").toAbsolutePath().normalize(), + Paths.get(engine.getDataFileMetaData().getTempDataFilePath()) + .toAbsolutePath().normalize()); + } + } + + @Test + public void dataFile_Relative_WatcherLoadsANewFile() throws Exception { + AtomicReference watcher = new AtomicReference<>(); + FutureFactory futureFactory = mock(FutureFactory.class); + when(futureFactory.schedule(any(Runnable.class), anyLong())) + .then(invocation -> mock(ScheduledFuture.class)); + when(futureFactory.scheduleRepeating(any(Runnable.class), anyLong())) + .then(invocation -> { + watcher.set(invocation.getArgument(0)); + return mock(ScheduledFuture.class); + }); + DataUpdateServiceDefault dataUpdate = new DataUpdateServiceDefault( + loggerFactory.getLogger(DataUpdateServiceDefault.class.getName()), + mock(HttpClient.class), + new FileWrapperFactoryDefault(), + futureFactory); + + try (TextFileEngine engine = + new TextFileEngineBuilder(loggerFactory, dataUpdate) + .setAutoUpdate(false) + .setDataFileSystemWatcher(true) + .setUpdatePollingIntervalSeconds(1) + .build(dataFile, false)) { + assertNotNull( + "A watcher is set up for the relative path", + engine.getDataFileMetaData().getConfiguration().getWatchKey()); + assertNotNull(watcher.get()); + + Files.write( + Paths.get(dataFile), + "New".getBytes(StandardCharsets.UTF_8)); + long end = System.currentTimeMillis() + WAIT_FOR_EVENT_MILLIS; + while ("New".equals(engine.getText()) == false && + System.currentTimeMillis() < end) { + watcher.get().run(); + Thread.sleep(RETRY_MILLIS); + } + + assertEquals("New", engine.getText()); + dataUpdate.unregisterDataFile(engine.getDataFileMetaData()); + } + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/performance/PipelineOverheadTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/performance/PipelineOverheadTests.java index 409ddd278..cff4b768f 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/performance/PipelineOverheadTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/performance/PipelineOverheadTests.java @@ -114,14 +114,21 @@ public void PipelineOverhead_Cache() throws Exception { CacheConfiguration cacheConfig = new CacheConfiguration( new LruPutCache.Builder(),100); engine.setCache(new FlowCacheDefault(cacheConfig)); - // Set process cost to 0.2 sec. Therefore, the test cannot pass - // unless the cache is mitigating this cost as it should do. + // Set process cost to 0.2 sec, so every call the cache does not answer + // costs far more than the overhead permitted. engine.setProcessCost(200); int iterations = 10_000; Map evidence = new HashMap<>(); evidence.put("test.value", 10); + // Fill the cache before timing. The one call that misses the cache + // costs 0.2 sec, which is 0.02 ms of the permitted average over + // 10,000 calls, so timing it measured the process cost and not the + // overhead. + new TestCallable(pipeline, 1, evidence).call(); long result = new TestCallable(pipeline, iterations, evidence).call(); + // The cache answered every call after the first. + assertEquals(1, engine.getProcessCount()); double msOverheadPerCall = (double)(result) / iterations; logger.info("Process cost {}", engine.getProcessCost()); logger.info("Average was {} millis", msOverheadPerCall); diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceBackgroundErrorTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceBackgroundErrorTests.java new file mode 100644 index 000000000..93863e58a --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceBackgroundErrorTests.java @@ -0,0 +1,148 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import ch.qos.logback.classic.Level; +import fiftyone.common.testhelpers.TestLogger; +import fiftyone.common.wrappers.io.FileWrapperFactory; +import fiftyone.pipeline.engines.configuration.DataFileConfigurationDefault; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectEngineDataFileDefault; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import fiftyone.pipeline.engines.services.update.FutureFactory; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ScheduledFuture; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * An unexpected error in the background update check is logged and kept + * inside the task, so it can neither end the thread silently nor reach the + * process. See + * exception-handling.md line 83: + * "When errors do occur, they will be logged and MUST NOT cause the + * process to crash." + *

    + * The scheduled task is captured and run on the test's thread, so nothing + * depends on timing. + */ +public class DataUpdateServiceBackgroundErrorTests { + + private TestLogger logger; + private FutureFactory futureFactory; + private FileWrapperFactory fileWrapperFactory; + private DataUpdateServiceDefault dataUpdate; + + @Before + public void init() { + ch.qos.logback.classic.Logger real = + (ch.qos.logback.classic.Logger) LoggerFactory.getLogger( + getClass()); + real.setLevel(Level.INFO); + logger = new TestLogger("test", real); + futureFactory = mock(FutureFactory.class); + doReturn(mock(ScheduledFuture.class)).when(futureFactory) + .schedule(any(Runnable.class), anyLong()); + doReturn(mock(ScheduledFuture.class)).when(futureFactory) + .scheduleRepeating(any(Runnable.class), anyLong()); + fileWrapperFactory = mock(FileWrapperFactory.class); + dataUpdate = new DataUpdateServiceDefault( + logger, + mock(HttpClient.class), + fileWrapperFactory, + futureFactory); + } + + @SuppressWarnings("unchecked") + private AspectEngineDataFileDefault registerFile() { + DataFileConfigurationDefault config = + new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(true); + config.setFileSystemWatcherEnabled(false); + config.setDataFilePath("data.dat"); + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(mock(OnPremiseAspectEngine.class)); + file.setConfiguration(config); + file.setTempDataFilePath("temp.dat"); + dataUpdate.registerDataFile(file); + return file; + } + + private Runnable scheduledCheck() { + ArgumentCaptor task = ArgumentCaptor.forClass(Runnable.class); + verify(futureFactory, atLeastOnce()).schedule(task.capture(), anyLong()); + return task.getValue(); + } + + @Test + public void scheduledCheck_UnexpectedError_IsLoggedAndKeptInTheTask() { + registerFile(); + Runnable check = scheduledCheck(); + when(fileWrapperFactory.getLastModified(anyString())) + .thenThrow(new IllegalStateException("disk went away")); + int errorsBefore = logger.errorsLogged.size(); + + try { + check.run(); + } catch (RuntimeException e) { + fail("The error left the background task: " + e); + } + + assertFalse("The error was not logged.", + logger.errorsLogged.size() == errorsBefore); + } + + @Test + public void scheduledCheck_ErrorInACallback_IsLoggedAndKeptInTheTask() { + dataUpdate.onUpdateComplete((sender, args) -> { + throw new IllegalStateException("callback failed"); + }); + registerFile(); + Runnable check = scheduledCheck(); + when(fileWrapperFactory.getLastModified(anyString())).thenReturn(0L); + int errorsBefore = logger.errorsLogged.size(); + + try { + check.run(); + } catch (RuntimeException e) { + fail("The error left the background task: " + e); + } + + assertFalse("The error was not logged.", + logger.errorsLogged.size() == errorsBefore); + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceCheckFailureTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceCheckFailureTests.java new file mode 100644 index 000000000..22de3ea5a --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceCheckFailureTests.java @@ -0,0 +1,586 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import fiftyone.common.wrappers.data.BinaryReader; +import fiftyone.common.wrappers.data.BinaryWriter; +import fiftyone.common.wrappers.io.FileWrapper; +import fiftyone.common.wrappers.io.FileWrapperDefault; +import fiftyone.common.wrappers.io.FileWrapperFactory; +import fiftyone.pipeline.engines.configuration.DataFileConfigurationDefault; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectEngineDataFileDefault; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import fiftyone.pipeline.engines.services.DataUpdateService.AutoUpdateStatus; +import fiftyone.pipeline.engines.services.update.FutureFactory; +import org.junit.Test; +import org.slf4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assume.assumeFalse; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockingDetails; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * A failure in a check started by update polling or by the file watcher is + * caught and logged at Error rather than lost with the scheduled task, is + * reported to the update callbacks unless one of them raised it, and does + * not stop later checks. See the Errors section of the data update + * specification. + */ +public class DataUpdateServiceCheckFailureTests { + + private static final long WAIT_FOR_EVENT_MILLIS = 10000; + + private static final long RETRY_MILLIS = 100; + + /** + * How far past now the second data file's modified time is set, so it + * is later than the first on a file system that keeps whole seconds. + */ + private static final long LATER_FILE_MILLIS = 2000; + + /** + * Modified times for a data file that is newer than the copy the + * engine uses. + */ + private static final long NEWER_MILLIS = 2000000; + + private static final long OLDER_MILLIS = 1000000; + + private static final String MAC_MESSAGE = + "File watchers are not well implemented in OS X, " + + "so don't run this unit test as it is unlikely to pass."; + + /** + * The Errors table's message for an error no other row covers, and for + * an error raised by an update callback. + */ + private static final String UNHANDLED = + "An unhandled error occurred while checking for automatic updates " + + "for engine '%s'"; + + /** + * A callback that fails on a check by the file watcher is logged, and + * the watcher goes on to apply a later data file. A repeating task that + * fails is never run again, so the failure must not leave the task. + */ + @Test + public void WatcherCheck_CallbackFails_LoggedAndWatchingCarriesOn() + throws Exception { + assumeFalse(MAC_MESSAGE, isMac()); + Logger logger = mock(Logger.class); + AtomicReference watcher = new AtomicReference<>(); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + logger, + mock(HttpClient.class), + fileWrapperFactory(), + futureFactory(watcher)); + RuntimeException failure = + new IllegalStateException("The callback failed"); + dataUpdate.onUpdateComplete((sender, args) -> { + throw failure; + }); + + OnPremiseAspectEngine engine = engine(); + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + AspectEngineDataFileDefault file = dataFile(engine, dataFile, true); + + try { + dataUpdate.registerDataFile(file); + assertNotNull("Registering sets up the file watcher", + watcher.get()); + + Files.write(dataFile, "New".getBytes()); + runWatcherUntilRefreshed(watcher.get(), engine, 1); + assertEquals("The watched file refreshes the engine", + 1, refreshCount(engine)); + verify(logger, atLeastOnce()).error( + eq(String.format(UNHANDLED, engineType(engine))), + same(failure)); + + Files.write(dataFile, "Newer".getBytes()); + Files.setLastModifiedTime(dataFile, FileTime.fromMillis( + System.currentTimeMillis() + LATER_FILE_MILLIS)); + runWatcherUntilRefreshed(watcher.get(), engine, 2); + assertEquals("A later data file is still applied", + 2, refreshCount(engine)); + } + finally { + cleanUp(file, directory, dataFile); + } + } + + /** + * A check by the file watcher whose refresh throws an Error still leaves + * the data file with a polling timer. The refresh cancels the timer + * before it runs, so the file has to be registered again however the + * refresh ends, or polling for updates never runs again. + */ + @Test + public void WatcherCheck_RefreshThrowsError_PollingTimerSetAgain() + throws Exception { + assumeFalse(MAC_MESSAGE, isMac()); + Logger logger = mock(Logger.class); + AtomicReference watcher = new AtomicReference<>(); + AtomicInteger pollingTimers = new AtomicInteger(); + FutureFactory futureFactory = mock(FutureFactory.class); + when(futureFactory.schedule(any(Runnable.class), anyLong())) + .then(invocation -> { + pollingTimers.incrementAndGet(); + return mock(ScheduledFuture.class); + }); + when(futureFactory.scheduleRepeating(any(Runnable.class), anyLong())) + .then(invocation -> { + watcher.set((Runnable) invocation.getArgument(0)); + return mock(ScheduledFuture.class); + }); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + logger, + mock(HttpClient.class), + fileWrapperFactory(), + futureFactory); + + OnPremiseAspectEngine engine = engine(); + doThrow(new Error("The refresh failed")) + .when(engine).refreshData(any()); + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + AspectEngineDataFileDefault file = dataFile(engine, dataFile, true); + + try { + dataUpdate.registerDataFile(file); + assertEquals("Registering sets one polling timer", + 1, pollingTimers.get()); + + Files.write(dataFile, "New".getBytes()); + runWatcherUntilRefreshed(watcher.get(), engine, 1); + assertEquals("The watched file was refreshed once", + 1, refreshCount(engine)); + assertEquals("A new polling timer is set after the failed refresh", + 2, pollingTimers.get()); + } + finally { + cleanUp(file, directory, dataFile); + } + } + + /** + * A callback that fails on a check by update polling is logged rather + * than left with the timer's task, and polling goes on. + */ + @Test + public void PollingCheck_CallbackFails_LoggedAndPollingCarriesOn() + throws Exception { + Logger logger = mock(Logger.class); + List timers = new ArrayList<>(); + HttpURLConnection connection = mock(HttpURLConnection.class); + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + HttpClient httpClient = mock(HttpClient.class); + when(httpClient.connect(any(URL.class))).thenReturn(connection); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + logger, + httpClient, + fileWrapperFactory(), + timerFactory(timers)); + RuntimeException failure = + new IllegalStateException("The callback failed"); + List statuses = new ArrayList<>(); + dataUpdate.onUpdateComplete((sender, args) -> { + statuses.add(args.getStatus()); + throw failure; + }); + + OnPremiseAspectEngine engine = engine(); + Path directory = Files.createTempDirectory("polled"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + AspectEngineDataFileDefault file = dataFile(engine, dataFile, false); + file.setTempDataDirPath(directory.toString()); + file.getConfiguration().setDataUpdateUrl("http://www.test.com"); + file.getConfiguration().setVerifyModifiedSince(false); + + try { + dataUpdate.registerDataFile(file); + assertEquals("Registering sets one polling timer", + 1, timers.size()); + + timers.get(0).run(); + + assertEquals("The check is reported once", + Collections.singletonList( + AutoUpdateStatus.AUTO_UPDATE_NOT_NEEDED), + statuses); + verify(logger).error( + eq(String.format(UNHANDLED, engineType(engine))), + same(failure)); + assertEquals("The next check is scheduled", 2, timers.size()); + } + finally { + cleanUp(file, directory, dataFile); + } + } + + /** + * A check by update polling that fails is logged, reported as an + * unknown error, and the next check is still scheduled, so a later data + * file is applied. + */ + @Test + public void PollingCheck_CheckFails_ReportedAndNextCheckScheduled() + throws Exception { + Logger logger = mock(Logger.class); + List timers = new ArrayList<>(); + RuntimeException failure = + new IllegalStateException("The file system failed"); + FileWrapperFactory fileWrapperFactory = fileWrapperFactory(); + when(fileWrapperFactory.getLastModified(anyString())) + .thenThrow(failure); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + logger, + mock(HttpClient.class), + fileWrapperFactory, + timerFactory(timers)); + List statuses = new ArrayList<>(); + dataUpdate.onUpdateComplete((sender, args) -> + statuses.add(args.getStatus().name())); + + OnPremiseAspectEngine engine = engine(); + Path directory = Files.createTempDirectory("polled"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + AspectEngineDataFileDefault file = dataFile(engine, dataFile, false); + file.setTempDataFilePath( + directory.resolve("data-copy.dat").toString()); + + try { + dataUpdate.registerDataFile(file); + assertEquals("Registering sets one polling timer", + 1, timers.size()); + + timers.get(0).run(); + + verify(logger).error( + eq(String.format(UNHANDLED, engineType(engine))), + same(failure)); + assertEquals("The failed check is reported as an unknown error", + Collections.singletonList("AUTO_UPDATE_UNKNOWN_ERROR"), + statuses); + assertEquals("The next check is scheduled", 2, timers.size()); + } + finally { + cleanUp(file, directory, dataFile); + } + } + + /** + * An Error thrown by the engine's refresh during a check by update + * polling is caught like any other failure. It is logged, reported as + * an unknown error, and the next check is scheduled. + */ + @Test + public void PollingCheck_RefreshThrowsError_ReportedAndNextCheckScheduled() + throws Exception { + Logger logger = mock(Logger.class); + List timers = new ArrayList<>(); + Error failure = new Error("The refresh failed"); + Path directory = Files.createTempDirectory("polled"); + Path dataFile = directory.resolve("data.dat"); + Path tempFile = directory.resolve("data-copy.dat"); + Files.write(dataFile, "New".getBytes()); + Files.write(tempFile, "Old".getBytes()); + FileWrapperFactory fileWrapperFactory = fileWrapperFactory(); + when(fileWrapperFactory.getLastModified(dataFile.toString())) + .thenReturn(NEWER_MILLIS); + when(fileWrapperFactory.getLastModified(tempFile.toString())) + .thenReturn(OLDER_MILLIS); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + logger, + mock(HttpClient.class), + fileWrapperFactory, + timerFactory(timers)); + List statuses = new ArrayList<>(); + dataUpdate.onUpdateComplete((sender, args) -> + statuses.add(args.getStatus().name())); + + OnPremiseAspectEngine engine = engine(); + doThrow(failure).when(engine).refreshData(any()); + AspectEngineDataFileDefault file = dataFile(engine, dataFile, false); + file.setTempDataFilePath(tempFile.toString()); + + try { + dataUpdate.registerDataFile(file); + assertEquals("Registering sets one polling timer", + 1, timers.size()); + + timers.get(0).run(); + + verify(logger).error( + eq(String.format(UNHANDLED, engineType(engine))), + same(failure)); + assertEquals("The failed check is reported as an unknown error", + Collections.singletonList("AUTO_UPDATE_UNKNOWN_ERROR"), + statuses); + assertEquals("The next check is scheduled", 2, timers.size()); + } + finally { + if (tempFile.toFile().delete() == false) { + tempFile.toFile().deleteOnExit(); + } + cleanUp(file, directory, dataFile); + } + } + + /** + * A programmatic check that fails is reported as an unknown error, and + * the error then passes to the caller without being logged, as the + * Programmatic update steps of the data update specification say. + */ + @Test + public void ProgrammaticCheck_CheckFails_ReportedAndPassedToCaller() + throws Exception { + Logger logger = mock(Logger.class); + RuntimeException failure = + new IllegalStateException("The file system failed"); + FileWrapperFactory fileWrapperFactory = fileWrapperFactory(); + when(fileWrapperFactory.getLastModified(anyString())) + .thenThrow(failure); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + logger, + mock(HttpClient.class), + fileWrapperFactory, + timerFactory(new ArrayList<>())); + List statuses = new ArrayList<>(); + dataUpdate.onUpdateComplete((sender, args) -> + statuses.add(args.getStatus().name())); + + OnPremiseAspectEngine engine = engine(); + Path directory = Files.createTempDirectory("checked"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + AspectEngineDataFileDefault file = dataFile(engine, dataFile, false); + file.getConfiguration().setAutomaticUpdatesEnabled(false); + file.setTempDataFilePath( + directory.resolve("data-copy.dat").toString()); + doReturn(file).when(engine).getDataFileMetaData(); + + try { + RuntimeException thrown = assertThrows(RuntimeException.class, + () -> dataUpdate.checkForUpdate(engine)); + + assertSame("The check's error passes to the caller", + failure, thrown); + assertEquals("The failed check is reported as an unknown error", + Collections.singletonList("AUTO_UPDATE_UNKNOWN_ERROR"), + statuses); + verify(logger, never()).error(anyString(), any(Throwable.class)); + } + finally { + cleanUp(file, directory, dataFile); + } + } + + private static void runWatcherUntilRefreshed( + Runnable watcher, + Object engine, + long count) throws InterruptedException { + long end = System.currentTimeMillis() + WAIT_FOR_EVENT_MILLIS; + while (refreshCount(engine) < count && + System.currentTimeMillis() < end) { + watcher.run(); + Thread.sleep(RETRY_MILLIS); + } + } + + private static FutureFactory futureFactory( + final AtomicReference watcher) { + FutureFactory futureFactory = mock(FutureFactory.class); + when(futureFactory.schedule(any(Runnable.class), anyLong())) + .then(invocation -> mock(ScheduledFuture.class)); + when(futureFactory.scheduleRepeating(any(Runnable.class), anyLong())) + .then(invocation -> { + watcher.set((Runnable) invocation.getArgument(0)); + return mock(ScheduledFuture.class); + }); + return futureFactory; + } + + private static FutureFactory timerFactory(final List timers) { + FutureFactory futureFactory = mock(FutureFactory.class); + when(futureFactory.schedule(any(Runnable.class), anyLong())) + .then(invocation -> { + timers.add((Runnable) invocation.getArgument(0)); + return mock(ScheduledFuture.class); + }); + when(futureFactory.scheduleRepeating(any(Runnable.class), anyLong())) + .then(invocation -> mock(ScheduledFuture.class)); + return futureFactory; + } + + private static FileWrapperFactory fileWrapperFactory() { + FileWrapperFactory fileWrapperFactory = mock(FileWrapperFactory.class); + when(fileWrapperFactory.build(anyString())).then(invocation -> + new InMemoryReadFile((String) invocation.getArgument(0))); + when(fileWrapperFactory.getLastModified(anyString())).then( + invocation -> + new File((String) invocation.getArgument(0)).lastModified()); + return fileWrapperFactory; + } + + /** + * A file whose reader holds a copy of its bytes. The default reader maps + * the file, and Windows will not let a mapped file be written again + * until the mapping is collected, which the watcher tests need to do. + */ + private static final class InMemoryReadFile implements FileWrapper { + private final FileWrapperDefault file; + + InMemoryReadFile(String path) { + file = new FileWrapperDefault(path); + } + + @Override + public BinaryReader getReader() throws IOException { + return new BinaryReader(ByteBuffer.wrap( + Files.readAllBytes(Paths.get(file.getPath())))); + } + + @Override + public BinaryWriter getWriter() throws IOException { + return file.getWriter(); + } + + @Override + public long getLastModified() { + return file.getLastModified(); + } + + @Override + public String getPath() { + return file.getPath(); + } + + @Override + public boolean exists() { + return file.exists(); + } + + @Override + public void delete() { + file.delete(); + } + } + + @SuppressWarnings("unchecked") + private static OnPremiseAspectEngine engine() { + return mock(OnPremiseAspectEngine.class); + } + + private static String engineType(Object engine) { + return engine.getClass().getSimpleName(); + } + + private static AspectEngineDataFileDefault dataFile( + OnPremiseAspectEngine engine, + Path dataFile, + boolean watching) { + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(true); + config.setFileSystemWatcherEnabled(watching); + config.setPollingIntervalSeconds(1); + config.setDataFilePath(dataFile.toString()); + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(engine); + file.setConfiguration(config); + return file; + } + + private static void cleanUp( + AspectEngineDataFileDefault file, + Path directory, + Path dataFile) { + if (file.getConfiguration().getWatchKey() != null) { + file.getConfiguration().getWatchKey().cancel(); + } + // Whatever Windows still holds is deleted when the test run ends, + // in the reverse order it is registered, so the folder goes first. + directory.toFile().deleteOnExit(); + if (dataFile.toFile().delete() == false) { + dataFile.toFile().deleteOnExit(); + } + directory.toFile().delete(); + } + + private static boolean isMac() { + return System.getProperty("os.name").contains("Mac OS X"); + } + + private static long refreshCount(Object engine) { + return mockingDetails(engine).getInvocations().stream() + .filter(i -> i.getMethod().getName().equals("refreshData")) + .count(); + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java index dcd972ae1..6f94e3e57 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java @@ -24,6 +24,7 @@ import ch.qos.logback.classic.Level; import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.IThrowableProxy; import ch.qos.logback.classic.spi.ThrowableProxyUtil; import ch.qos.logback.core.read.ListAppender; import fiftyone.common.wrappers.io.FileWrapperFactoryDefault; @@ -41,8 +42,11 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; import java.net.HttpURLConnection; import java.net.URL; +import java.security.MessageDigest; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -50,11 +54,14 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -76,6 +83,8 @@ public class DataUpdateServiceLoggingTests { FULL_KEY + "&Download=True&Type=HashV41"; private static final String MASKED_URL = UPDATE_URL + "?LicenseKeys=" + MASKED_KEY + "&Download=True&Type=HashV41"; + // The body of a download the tests apply. + private static final byte[] DOWNLOAD = {1, 2, 3}; private ch.qos.logback.classic.Logger logger; private ListAppender appender; @@ -290,20 +299,18 @@ public void FileSystemWatcher_LogsCreatingFileSystemWatcher() { } @Test - public void ConnectFails_Manual_WarnsWithoutRetryText() throws Exception { + public void ConnectFails_Manual_LogsTheAccessError() throws Exception { when(httpClient.connect(any(URL.class))) .thenThrow(new IOException("refused")); newDataFile(false, false, false); dataUpdate.checkForUpdate(engine); - assertLogged(Level.WARN, "An error occurred when connecting to " + - UPDATE_URL + " in order to check for data file updates for " + - engineType + ". Error detail: refused"); + assertCheckFailed(accessFailure(UPDATE_URL), "refused"); } @Test - public void ConnectFails_Automatic_WarnsThatUpdateWillBeTriedAgain() + public void ConnectFails_Automatic_LogsTheAccessError() throws Exception { when(httpClient.connect(any(URL.class))) .thenThrow(new IOException("refused")); @@ -311,14 +318,11 @@ public void ConnectFails_Automatic_WarnsThatUpdateWillBeTriedAgain() dataUpdate.registerDataFile(file); - assertLogged(Level.WARN, "An error occurred when connecting to " + - UPDATE_URL + " in order to check for data file updates for " + - engineType + ". Update will be attempted again later. " + - "Error detail: refused"); + assertCheckFailed(accessFailure(UPDATE_URL), "refused"); } @Test - public void ConnectFails_AutoUpdateOff_DoesNotPromiseAnotherTry() + public void ConnectFails_AutoUpdateOff_LogsTheAccessError() throws Exception { when(httpClient.connect(any(URL.class))) .thenThrow(new IOException("refused")); @@ -326,37 +330,63 @@ public void ConnectFails_AutoUpdateOff_DoesNotPromiseAnotherTry() dataUpdate.registerDataFile(file); - assertLogged(Level.WARN, "An error occurred when connecting to " + - UPDATE_URL + " in order to check for data file updates for " + - engineType + ". Error detail: refused"); + assertCheckFailed(accessFailure(UPDATE_URL), "refused"); } - private void assertStatusWarned(int status, String detail) + /** + * A 200 response whose body ends before all of it has arrived is the + * first row of the Errors table, as a failure to connect is. + */ + @Test + public void DownloadCutShort_LogsTheAccessError() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(connection.getInputStream()).thenReturn(new InputStream() { + private int sent = 0; + + @Override + public int read() throws IOException { + if (sent < 10) { + sent++; + return 'x'; + } + throw new IOException("Premature EOF"); + } + }); + newDataFile(false, false, false); + + assertEquals(DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR, + dataUpdate.checkForUpdate(engine)); + + assertCheckFailed(accessFailure(UPDATE_URL), "Premature EOF"); + } + + private void assertStatusFailed(int status, String message) throws Exception { when(connection.getResponseCode()).thenReturn(status); newDataFile(true, false, false); dataUpdate.checkForUpdate(engine); - assertLogged(Level.WARN, "An error occurred while downloading a data " + - "file update for " + engineType + " from " + UPDATE_URL + ". " + - "Error detail: " + detail); - assertTrue(logged(Level.ERROR).isEmpty()); + assertCheckFailed(message, null); } @Test - public void TooManyRequests_WarnsAboutTheDownload() throws Exception { - assertStatusWarned(429, "HTTP status code 429 (too many requests)"); + public void TooManyRequests_LogsTheTableRow() throws Exception { + assertStatusFailed(429, "Too many requests to '" + UPDATE_URL + + "' for engine '" + engineType + "'"); } @Test - public void Forbidden_WarnsAboutTheDownload() throws Exception { - assertStatusWarned(403, "HTTP status code 403 (access denied)"); + public void Forbidden_LogsTheTableRow() throws Exception { + assertStatusFailed(403, "Access denied to data update service at '" + + UPDATE_URL + "' for engine '" + engineType + "'"); } @Test - public void OtherStatus_WarnsAboutTheDownload() throws Exception { - assertStatusWarned(500, "HTTP status code 500"); + public void OtherStatus_LogsTheTableRow() throws Exception { + assertStatusFailed(500, "HTTP status code '500' from data update " + + "service at '" + UPDATE_URL + "' for engine '" + engineType + "'"); } /** @@ -393,33 +423,136 @@ public void TemporaryFileCannotBeWritten_LogsTheWriteError() } @Test - public void Md5DoesNotMatch_WarnsAboutIntegrityCheck() throws Exception { + public void Md5DoesNotMatch_LogsTheTableRow() throws Exception { when(connection.getResponseCode()) .thenReturn(HttpURLConnection.HTTP_OK); when(connection.getInputStream()) - .thenReturn(new ByteArrayInputStream(new byte[]{1, 2, 3})); + .thenReturn(new ByteArrayInputStream(DOWNLOAD)); when(connection.getHeaderField("Content-MD5")).thenReturn("0"); AspectEngineDataFileDefault file = newDataFile(true, false, false); file.getConfiguration().setVerifyMd5(true); - dataUpdate.checkForUpdate(engine); + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); - assertLogged(Level.WARN, "An error occurred during the integrity " + - "check of new data file for " + engineType + ". Error detail: " + - "the MD5 hash of the download does not match the Content-MD5 " + - "header"); + assertEquals( + DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED, + status); + assertCheckFailed("Integrity check failed. MD5 hash in HTTP " + + "response '0' for '" + engineType + "'data update does not " + + "match calculated hash for the downloaded file '" + + md5Hex(DOWNLOAD) + "'.", null); + verify(engine, never()).refreshData(anyString(), any(byte[].class)); } + /** + * A response with no Content-MD5 header is warned about, and the check + * goes on to apply the download. + */ @Test - public void NoTemporaryFilePath_LogsError() { + public void Md5Missing_WarnsAndAppliesTheUpdate() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(connection.getInputStream()) + .thenReturn(new ByteArrayInputStream(DOWNLOAD)); + AspectEngineDataFileDefault file = newDataFile(true, false, false); + file.getConfiguration().setVerifyMd5(true); + + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.WARN, "No MD5 hash included in data update " + + "response for '" + UPDATE_URL + "'. Unable to verify data " + + "integrity"); + assertEquals( + DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_SUCCESS, + status); + verify(engine).refreshData(anyString(), any(byte[].class)); + } + + @Test + public void NoTemporaryFilePath_LogsTheTableRow() { AspectEngineDataFileDefault file = newDataFile(false, false, false); file.setTempDataDirPath(null); when(engine.getTempDataDirPath()).thenReturn(null); - dataUpdate.checkForUpdate(engine); + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); + + assertEquals("AUTO_UPDATE_TEMP_PATH_NOT_SET", status.name()); + assertCheckFailed("The data file 'default' is checking for " + + "updates but does not have a temporary file path configured.", + null); + } + + /** + * Data passed in from memory that cannot be written to the data file is + * warned about, and the engine is still refreshed with it. + */ + @Test + public void UpdateFromMemory_FileCannotBeWritten_WarnsAndRefreshes() + throws Exception { + AspectEngineDataFileDefault file = newDataFile(false, false, false); + String unwritable = new File(new File( + System.getProperty("java.io.tmpdir"), + "no-such-directory-" + System.nanoTime()), "data.hash") + .getAbsolutePath(); + file.getConfiguration().setDataFilePath(unwritable); - assertLogged(Level.ERROR, "The data file 'default' is checking for " + - "updates but does not have a temporary file path configured."); + DataUpdateService.AutoUpdateStatus status = + dataUpdate.updateFromMemory(file, DOWNLOAD); + + assertLogged(Level.WARN, "An error occurred when writing to '" + + unwritable + "'. The engine will be updated to use the new data " + + "but the file on disk will still contain old data."); + assertTrue("An error was logged: " + logged(Level.ERROR), + logged(Level.ERROR).isEmpty()); + assertEquals( + DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_SUCCESS, + status); + verify(engine).refreshData(anyString(), any(byte[].class)); + } + + /** + * An error of any other kind is logged at Error with the Errors table's + * last message and the error attached, and gives + * AUTO_UPDATE_UNKNOWN_ERROR. + */ + @Test + public void UnexpectedError_LogsTheLastRow() { + AspectEngineDataFileDefault file = newDataFile(false, false, false); + // The engine's copy is the data file itself, so the file on disk is + // not newer and the URL is checked. + file.setTempDataFilePath(dataFile.getAbsolutePath()); + final IllegalStateException broke = + new IllegalStateException("formatter broke"); + file.getConfiguration().setUrlFormatter(d -> { + throw broke; + }); + + assertEquals("AUTO_UPDATE_UNKNOWN_ERROR", + dataUpdate.checkForUpdate(engine).name()); + + for (ILoggingEvent event : appender.list) { + if (event.getLevel() == Level.ERROR && + event.getFormattedMessage().equals("An unhandled error " + + "occurred while checking for automatic updates for " + + "engine '" + engineType + "'") && + event.getThrowableProxy() != null && + "formatter broke".equals( + event.getThrowableProxy().getMessage())) { + return; + } + } + fail("No unhandled error entry in " + logged(Level.ERROR)); + } + + /** + * The MD5 hash of the bytes as lower case hexadecimal. + */ + private static String md5Hex(byte[] bytes) throws Exception { + byte[] digest = MessageDigest.getInstance("MD5").digest(bytes); + return String.format("%032x", new BigInteger(1, digest)); } /** @@ -436,6 +569,46 @@ private AspectEngineDataFileDefault newKeyedDataFile(boolean inMemory) { return file; } + /** + * An Error row of the specification's Errors table, logged at Error as + * "Error during check for data update" with an error attached that + * carries the row's message, and a cause whose message holds the given + * text when there is one. A cause with a License Key masked out of it + * is a copy whose message starts with the original's class name. + */ + private void assertCheckFailed(String message, String causeMessage) { + List found = new ArrayList<>(); + for (ILoggingEvent event : appender.list) { + if (event.getLevel() == Level.ERROR && + "Error during check for data update".equals( + event.getFormattedMessage()) && + event.getThrowableProxy() != null) { + IThrowableProxy error = event.getThrowableProxy(); + found.add(error.getMessage()); + if (message.equals(error.getMessage())) { + IThrowableProxy cause = error.getCause(); + if (causeMessage == null) { + assertNull(cause); + } else { + assertNotNull(cause); + assertTrue("The cause was " + cause.getMessage(), + cause.getMessage().endsWith(causeMessage)); + } + assertTrue("A warning was logged as well: " + + logged(Level.WARN), logged(Level.WARN).isEmpty()); + return; + } + } + } + fail("Expected '" + message + "' attached to an Error entry, but " + + "the errors attached were " + found); + } + + private String accessFailure(String url) { + return "Error accessing data update service at '" + url + + "' for engine '" + engineType + "'"; + } + /** * Fails if any log entry, or any throwable logged with one, holds the * license key in full. @@ -498,9 +671,8 @@ public void LicenseKey_ConnectFails_MasksTheUrlAndTheException() dataUpdate.checkForUpdate(engine); - assertLogged(Level.WARN, "An error occurred when connecting to " + - MASKED_URL + " in order to check for data file updates for " + - engineType + ". Error detail: Connection refused: " + MASKED_URL); + assertCheckFailed(accessFailure(MASKED_URL), + "Connection refused: " + MASKED_URL); assertKeyOnlyMasked(); } @@ -515,9 +687,8 @@ public void LicenseKey_DownloadFails_MasksTheUrlAndTheException() dataUpdate.checkForUpdate(engine); - assertLogged(Level.WARN, "An error occurred while downloading a data " + - "file update for " + engineType + " from " + MASKED_URL + ". " + - "Error detail: Premature EOF from " + MASKED_URL); + assertCheckFailed(accessFailure(MASKED_URL), + "Premature EOF from " + MASKED_URL); assertKeyOnlyMasked(); } @@ -529,9 +700,8 @@ public void LicenseKey_Forbidden_LogsTheMaskedUrl() throws Exception { dataUpdate.checkForUpdate(engine); - assertLogged(Level.WARN, "An error occurred while downloading a data " + - "file update for " + engineType + " from " + MASKED_URL + ". " + - "Error detail: HTTP status code 403 (access denied)"); + assertCheckFailed("Access denied to data update service at '" + + MASKED_URL + "' for engine '" + engineType + "'", null); assertKeyOnlyMasked(); } @@ -542,10 +712,8 @@ public void LicenseKey_NoResponse_LogsTheMaskedUrl() throws Exception { dataUpdate.checkForUpdate(engine); - assertLogged(Level.WARN, "An error occurred when connecting to " + - MASKED_URL + " in order to check for data file updates for " + - engineType + ". Error detail: No response from the data update " + - "service"); + assertCheckFailed("No response from data update service at '" + + MASKED_URL + "' for engine '" + engineType + "'", null); assertKeyOnlyMasked(); } } diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java index b10b06a8a..50ec8f88f 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java @@ -289,6 +289,13 @@ public void DataUpdateService_Register_AutoUpdateConfiguredInterval() { file.setEngine(engine); file.setConfiguration(config); + // With no polling interval the delay is the random number of whole + // seconds alone, which can be 0. The check is not run, so a check + // that starts at once cannot schedule again before the verify. + doReturn(mock(ScheduledFuture.class)).when(futureFactory).schedule( + any(Runnable.class), + anyLong()); + // Act dataUpdate.registerDataFile(file); @@ -755,9 +762,9 @@ public void call(Object sender, DataUpdateService.DataUpdateCompleteArgs args) { } }); - // The invalid MD5 will cause a warning to be logged so we want - // to ignore this - ignoreWranings = 1; + // The invalid MD5 is the Errors table integrity row, logged once + // at Error, so it is ignored in cleanup. + ignoreErrors = 1; OnPremiseAspectEngine engine = mock(OnPremiseAspectEngine.class); DataFileConfigurationDefault config = new DataFileConfigurationDefault(); @@ -892,12 +899,13 @@ public void call(Object sender, DataUpdateService.DataUpdateCompleteArgs args) { // reconfigured to try again later. verify(futureFactory, atLeast(1)) .schedule(any(Runnable.class), anyLong()); - // We expect one warning to be logged so make sure it's - // ignored in cleanup and verify its presence. - ignoreWranings = 1; - assertEquals(1, logger.warningsLogged.size()); - assertTrue(logger.warningsLogged.get(0).contains(errorText)); - assertEquals(0, logger.errorsLogged.size()); + // The failed connection is the Errors table's access error, + // logged once at Error, so it is ignored in cleanup. + ignoreErrors = 1; + assertEquals(0, logger.warningsLogged.size()); + assertEquals(1, logger.errorsLogged.size()); + assertEquals("Error during check for data update", + logger.errorsLogged.get(0)); assertEquals(AUTO_UPDATE_HTTPS_ERR, completeEventArgs[0].getStatus()); } finally { @@ -1724,8 +1732,8 @@ public void call(Object sender, DataUpdateService.DataUpdateCompleteArgs args) { assertTrue("The 'checkForUpdateComplete' " + "event was never fired", completed); - // Ignore the warning that is logged due to the 429 - ignoreWranings = 1; + // Ignore the error that is logged due to the 429 + ignoreErrors = 1; // Check that the timer has been set to go off again. assertNotNull(file.getFuture()); // Check that the timer has been set to expire in 10 seconds. diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/HostLookupTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/HostLookupTests.java new file mode 100644 index 000000000..06bddbe6b --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/HostLookupTests.java @@ -0,0 +1,266 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import org.junit.After; +import org.junit.Test; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.net.ProxySelector; +import java.net.SocketAddress; +import java.net.SocketTimeoutException; +import java.net.URI; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Checks that a host lookup that does not answer fails within the connect + * timeout, as the connection's own timeouts do not cover the lookup. + */ +public class HostLookupTests { + + private static final String HOST = "unanswered.example"; + + /** The timeout given to each lookup. */ + private static final int TIMEOUT_MILLIS = 200; + + /** + * How long a resolver that does not answer holds a lookup unless the + * test releases it. It stands in for a system resolver that waits 30 + * seconds before giving up. + */ + private static final long UNANSWERED_MILLIS = 30000; + + /** + * The longest a timed out lookup may take. It is many times the timeout, + * so a slow runner passes, and far below the resolver's wait, so a + * lookup that waited for the resolver fails. + */ + private static final long LIMIT_MILLIS = 5000; + + /** The longest a test waits for another thread to reach a point. */ + private static final long WAIT_MILLIS = 10000; + + private final CountDownLatch release = new CountDownLatch(1); + private final CountDownLatch entered = new CountDownLatch(1); + private final AtomicInteger calls = new AtomicInteger(); + + @After + public void releaseResolver() { + release.countDown(); + } + + /** + * A resolver that holds each lookup until released, then answers with + * the loopback address, or fails if it is never released. + */ + private HostLookup.Resolver held() { + return host -> { + calls.incrementAndGet(); + entered.countDown(); + try { + if (release.await(UNANSWERED_MILLIS, TimeUnit.MILLISECONDS)) { + return new InetAddress[] { InetAddress.getLoopbackAddress() }; + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + throw new UnknownHostException(host); + }; + } + + private static URL url() throws IOException { + return URI.create("https://" + HOST + "/api/v4/evidencekeys").toURL(); + } + + private static long millisSince(long start) { + return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); + } + + @Test + public void LookUp_ResolverDoesNotAnswer_TimesOutNamingTheHost() + throws Exception { + HostLookup lookup = new HostLookup(held()); + long start = System.nanoTime(); + try { + lookup.lookUp(url(), TIMEOUT_MILLIS); + fail("The lookup should have timed out."); + } catch (SocketTimeoutException e) { + long took = millisSince(start); + assertTrue("The lookup took " + took + " ms.", took < LIMIT_MILLIS); + assertEquals( + String.format(HostLookup.MessageLookupTimedOut, + HOST, TIMEOUT_MILLIS), + e.getMessage()); + } + } + + @Test + public void LookUp_UnknownHost_ThrowsUnknownHostWithTheCause() + throws Exception { + UnknownHostException unknown = new UnknownHostException(HOST); + HostLookup lookup = new HostLookup(host -> { + throw unknown; + }); + try { + lookup.lookUp(url(), TIMEOUT_MILLIS); + fail("The lookup should have failed."); + } catch (UnknownHostException e) { + assertEquals(HOST, e.getMessage()); + assertSame(unknown, e.getCause()); + } + } + + @Test + public void LookUp_NoTimeout_DoesNotLookUp() throws Exception { + new HostLookup(held()).lookUp(url(), 0); + assertEquals(0, calls.get()); + } + + @Test + public void LookUp_AfterAFailure_LooksUpAgain() throws Exception { + HostLookup lookup = new HostLookup(host -> { + calls.incrementAndGet(); + throw new UnknownHostException(host); + }); + for (int i = 0; i < 2; i++) { + try { + lookup.lookUp(url(), TIMEOUT_MILLIS); + fail("The lookup should have failed."); + } catch (UnknownHostException e) { + // Expected. + } + } + assertEquals(2, calls.get()); + } + + @Test + public void LookUp_SameHostAtOnce_SharesOneLookup() throws Exception { + HostLookup lookup = new HostLookup(held()); + AtomicReference firstFailure = new AtomicReference<>(); + AtomicReference secondFailure = new AtomicReference<>(); + Thread first = lookUpOnThread(lookup, firstFailure); + assertTrue(entered.await(WAIT_MILLIS, TimeUnit.MILLISECONDS)); + Thread second = lookUpOnThread(lookup, secondFailure); + // The second caller is waiting on the first one's lookup once it is + // in a timed wait. + long start = System.nanoTime(); + while (second.getState() != Thread.State.TIMED_WAITING && + millisSince(start) < WAIT_MILLIS) { + Thread.sleep(10); + } + release.countDown(); + first.join(WAIT_MILLIS); + second.join(WAIT_MILLIS); + assertNull(firstFailure.get()); + assertNull(secondFailure.get()); + assertEquals(1, calls.get()); + } + + private static Thread lookUpOnThread( + HostLookup lookup, + AtomicReference failure) { + Thread thread = new Thread(() -> { + try { + lookup.lookUp(url(), (int) WAIT_MILLIS); + } catch (Throwable e) { + failure.set(e); + } + }); + thread.start(); + return thread; + } + + @Test + public void LookUp_ThroughAProxy_DoesNotLookUp() throws Exception { + ProxySelector original = ProxySelector.getDefault(); + ProxySelector.setDefault(new ProxySelector() { + @Override + public List select(URI uri) { + return Collections.singletonList(new Proxy( + Proxy.Type.HTTP, + InetSocketAddress.createUnresolved("proxy.example", 8080))); + } + + @Override + public void connectFailed( + URI uri, SocketAddress address, IOException e) { + } + }); + try { + new HostLookup(held()).lookUp(url(), TIMEOUT_MILLIS); + } finally { + ProxySelector.setDefault(original); + } + assertEquals(0, calls.get()); + } + + @Test + public void GetResponseString_HostDoesNotAnswer_FailsWithinConnectTimeout() + throws Exception { + HttpClient client = new HttpClientDefault(new HostLookup(held())); + HttpURLConnection connection = client.connect(url()); + connection.setConnectTimeout(TIMEOUT_MILLIS); + long start = System.nanoTime(); + try { + client.getResponseString(connection); + fail("The request should have timed out."); + } catch (SocketTimeoutException e) { + long took = millisSince(start); + assertTrue("The request took " + took + " ms.", took < LIMIT_MILLIS); + } + } + + @Test + public void PostData_HostDoesNotAnswer_FailsWithinConnectTimeout() + throws Exception { + HttpClient client = new HttpClientDefault(new HostLookup(held())); + HttpURLConnection connection = client.connect(url()); + connection.setConnectTimeout(TIMEOUT_MILLIS); + long start = System.nanoTime(); + try { + client.postData( + connection, Collections.emptyMap(), new byte[] { 1 }); + fail("The request should have timed out."); + } catch (SocketTimeoutException e) { + long took = millisSince(start); + assertTrue("The request took " + took + " ms.", took < LIMIT_MILLIS); + } + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyMessageTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyMessageTests.java new file mode 100644 index 000000000..8c902117a --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyMessageTests.java @@ -0,0 +1,211 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.data.AspectPropertyMetaDataDefault; +import fiftyone.pipeline.engines.flowelements.AspectEngine; +import fiftyone.pipeline.engines.flowelements.CloudAspectEngine; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Each missing property message is the text the specification's table of + * reasons gives, with campaign parameters allowed on each link. Lists in a + * message are joined with a comma and no space. + */ +public class MissingPropertyMessageTests { + + private static final String DOMAIN = "51degrees.com"; + + private static final String PRICING = "https://" + DOMAIN + "/pricing"; + + private static final String CONFIGURATOR = "https://configure." + DOMAIN; + + /** + * The campaign parameters on a link, up to the end of the link, which + * is a space, the end of the message, or a full stop that ends it. + */ + private static final Pattern CAMPAIGN = + Pattern.compile("\\?utm_\\S*?(?=\\.(\\s|$)|\\s|$)"); + + private static final String PROPERTY = "HardwareVendor"; + + private static final String DATA_KEY = "device"; + + private static final String PREFIX = "Property '" + PROPERTY + + "' not found in data for element '" + DATA_KEY + "'. "; + + private final MissingPropertyService service = + MissingPropertyServiceDefault.getInstance(); + + @Test + public void DataFileUpgradeRequired_TiersKnown() { + AspectEngine engine = + onPremise("Lite"); + listed(engine, Arrays.asList("Premium", "Enterprise"), true); + + assertMessage( + PREFIX + "This is because your license or data file does not " + + "include this property. It is available with the " + + "Premium,Enterprise license/data for the " + + engine.getClass().getSimpleName() + ". To upgrade, see " + + PRICING + ".", + engine); + } + + @Test + public void DataFileUpgradeRequired_TiersNotKnown() { + AspectEngine engine = + onPremise("Lite"); + listed(engine, Collections.emptyList(), true); + + assertMessage( + PREFIX + "This is because your license or data file does not " + + "include this property. To see which licenses include it, " + + "see " + PRICING + ".", + engine); + } + + @Test + public void PropertyExcludedFromEngineConfiguration() { + AspectEngine engine = + onPremise("Premium"); + listed(engine, Arrays.asList("Premium", "Enterprise"), false); + + assertMessage( + PREFIX + "This is because the property has been excluded when " + + "configuring the engine.", + engine); + } + + @Test + public void ProductNotAccessibleWithResourceKey() { + AspectEngine engine = + cloud(Collections.emptyList()); + + assertMessage( + PREFIX + "This is because your resource key does not include " + + "access to any properties under '" + DATA_KEY + "'. Check " + + "or create a resource key with the properties you need in " + + "the configurator at " + CONFIGURATOR + ".", + engine); + } + + @Test + public void PropertyNotAccessibleWithResourceKey() { + AspectEngine engine = + cloud(Arrays.asList("IsMobile", "IsTablet")); + + assertMessage( + PREFIX + "This is because your resource key does not include " + + "access to this property. The properties included for your " + + "key under '" + DATA_KEY + "' are IsMobile,IsTablet. Check " + + "or change your key in the configurator at " + + CONFIGURATOR + ".", + engine); + } + + @Test + public void Unknown() { + AspectEngine engine = + onPremise("Premium"); + when(engine.getProperties()).thenReturn( + Collections.emptyList()); + + assertMessage( + PREFIX + "The reason for this is unknown. Please check that the " + + "aspect and property name are correct.", + engine); + } + + private void assertMessage( + String expected, + AspectEngine engine) { + String message = service.getMissingPropertyReason(PROPERTY, engine) + .getDescription(); + assertEquals(expected, CAMPAIGN.matcher(message).replaceAll("")); + } + + @SuppressWarnings("unchecked") + private static AspectEngine + onPremise(String tier) { + OnPremiseAspectEngine + engine = mock(OnPremiseAspectEngine.class); + when(engine.getDataSourceTier()).thenReturn(tier); + when(engine.getElementDataKey()).thenReturn(DATA_KEY); + return engine; + } + + /** + * A Cloud Engine whose metadata is loaded and lists the given names. + */ + @SuppressWarnings("unchecked") + private static AspectEngine + cloud(List names) { + CloudAspectEngine + engine = mock(CloudAspectEngine.class); + when(engine.getElementDataKey()).thenReturn(DATA_KEY); + when(engine.hasLoadedProperties()).thenReturn(true); + List properties = new ArrayList<>(); + for (String name : names) { + properties.add(new AspectPropertyMetaDataDefault( + name, + engine, + "", + String.class, + Collections.emptyList(), + true)); + } + when(engine.getProperties()).thenReturn(properties); + return engine; + } + + /** + * The engine's metadata lists the property, in the given data tiers. + */ + private static void listed( + AspectEngine engine, + List tiers, + boolean available) { + AspectPropertyMetaData property = new AspectPropertyMetaDataDefault( + PROPERTY, + engine, + "", + String.class, + tiers, + available); + when(engine.getProperties()).thenReturn(Arrays.asList(property)); + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyReasonTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyReasonTests.java new file mode 100644 index 000000000..4f1123c9d --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyReasonTests.java @@ -0,0 +1,286 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import fiftyone.pipeline.core.typed.TypedKeyDefault; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.data.AspectPropertyMetaDataDefault; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.flowelements.AspectEngine; +import fiftyone.pipeline.engines.flowelements.CloudAspectEngine; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The reason a property is missing, decided for each engine in the order + * the specification gives. The data tier is compared for an On-premise + * Engine only. A Cloud Engine whose metadata is not loaded gives + * CloudRequestFailed when its request for the property list failed, and + * Unknown otherwise. An On-premise Engine whose metadata does not list the + * property, but whose Element Data has a typed accessor for it, gives + * DataFileUpgradeRequired with no data tiers. When every engine gives + * Unknown, the message names the last engine. + */ +public class MissingPropertyReasonTests { + + /** + * A Cloud Engine that says whether its request for the property list + * failed. + */ + public abstract static class PropertyListCloudEngine + implements CloudAspectEngine { + public abstract boolean propertyListRequestFailed(); + } + + /** + * Element Data whose typed accessor for HardwareVendor is inherited. + */ + public interface BaseDeviceData extends AspectData { + AspectPropertyValue getHardwareVendor(); + } + + public interface DeviceData extends BaseDeviceData { + } + + private static final String PREFIX = + "Property '%s' not found in data for element '%s'. "; + + private static final String CLOUD_REQUEST_FAILED = + "This is because an upstream cloud request failed, so no data was " + + "available to populate the property."; + + private static final String UNKNOWN = + "The reason for this is unknown. Please check that the aspect and " + + "property name are correct."; + + private static final Pattern CAMPAIGN = + Pattern.compile("\\?utm_\\S*?(?=\\.(\\s|$)|\\s|$)"); + + private final MissingPropertyService service = + MissingPropertyServiceDefault.getInstance(); + + @Test + public void CloudEngine_PropertyListed_NoDataTierCheck() { + CloudAspectEngine engine = cloud(); + listed(engine, "Premium", true); + + assertReason( + "PropertyNotAccessibleWithResourceKey", + service.getMissingPropertyReason("HardwareVendor", engine)); + } + + @Test + public void OtherEngine_PropertyListed_NoDataTierCheck() { + @SuppressWarnings("unchecked") + AspectEngine engine = + mock(AspectEngine.class); + when(engine.getElementDataKey()).thenReturn("device"); + when(engine.getDataSourceTier()).thenReturn("Lite"); + listed(engine, "Premium", true); + + assertReason( + "Unknown", + service.getMissingPropertyReason("HardwareVendor", engine)); + } + + @Test + public void OnPremiseEngine_PropertyListed_DataTierChecked() { + AspectEngine engine = onPremise(null); + listed(engine, "Premium", true); + + assertReason( + "DataFileUpgradeRequired", + service.getMissingPropertyReason("hardwarevendor", engine)); + } + + @Test + public void CloudEngine_PropertyListedNotAvailable_Excluded() { + CloudAspectEngine engine = cloud(); + listed(engine, "Premium", false); + + assertReason( + "PropertyExcludedFromEngineConfiguration", + service.getMissingPropertyReason("HardwareVendor", engine)); + } + + @Test + public void CloudEngine_MetadataNotLoaded_ListRequestFailed() { + PropertyListCloudEngine engine = notLoaded(true); + + MissingPropertyResult result = + service.getMissingPropertyReason("HardwareVendor", engine); + + assertReason("CloudRequestFailed", result); + assertEquals( + String.format(PREFIX, "HardwareVendor", "device") + + CLOUD_REQUEST_FAILED, + result.getDescription()); + } + + @Test + public void CloudEngine_MetadataNotLoaded_ListRequestNotFailed() { + PropertyListCloudEngine engine = notLoaded(false); + + assertReason( + "Unknown", + service.getMissingPropertyReason("HardwareVendor", engine)); + } + + @Test + public void OnPremiseEngine_NotListed_TypedAccessor_TiersNotKnown() { + AspectEngine engine = onPremise(DeviceData.class); + listed(engine, "Premium", true, "IsMobile"); + + MissingPropertyResult result = + service.getMissingPropertyReason("hardwarevendor", engine); + + assertReason("DataFileUpgradeRequired", result); + assertEquals( + String.format(PREFIX, "hardwarevendor", "device") + + "This is because your license or data file does not " + + "include this property. To see which licenses include it, " + + "see https://" + "51degrees.com/pricing.", + CAMPAIGN.matcher(result.getDescription()).replaceAll("")); + } + + @Test + public void OnPremiseEngine_NotListed_NoTypedAccessor_Unknown() { + AspectEngine engine = onPremise(DeviceData.class); + listed(engine, "Premium", true, "IsMobile"); + + assertReason( + "Unknown", + service.getMissingPropertyReason("PlatformName", engine)); + } + + @Test + public void CloudEngine_NotListed_TypedAccessor_NotAnUpgrade() { + CloudAspectEngine engine = cloud(); + doReturn(new TypedKeyDefault<>("device", DeviceData.class)) + .when(engine).getTypedDataKey(); + listed(engine, "Premium", true, "IsMobile"); + + assertReason( + "PropertyNotAccessibleWithResourceKey", + service.getMissingPropertyReason("HardwareVendor", engine)); + } + + @Test + public void EveryEngineUnknown_MessageNamesTheLastEngine() { + AspectEngine first = onPremise(null); + when(first.getElementDataKey()).thenReturn("first"); + listed(first, "Premium", true, "IsMobile"); + AspectEngine last = onPremise(null); + when(last.getElementDataKey()).thenReturn("last"); + listed(last, "Premium", true, "IsMobile"); + List> + engines = new ArrayList<>(); + engines.add(first); + engines.add(last); + + MissingPropertyResult result = + service.getMissingPropertyReason("PlatformName", engines); + + assertReason("Unknown", result); + assertEquals( + String.format(PREFIX, "PlatformName", "last") + UNKNOWN, + result.getDescription()); + } + + private static void assertReason( + String expected, + MissingPropertyResult result) { + assertEquals( + result.getDescription(), + expected, + result.getReason().name()); + } + + @SuppressWarnings("unchecked") + private static CloudAspectEngine cloud() { + CloudAspectEngine engine = + mock(CloudAspectEngine.class); + when(engine.getElementDataKey()).thenReturn("device"); + when(engine.getDataSourceTier()).thenReturn("Lite"); + when(engine.hasLoadedProperties()).thenReturn(true); + return engine; + } + + private static PropertyListCloudEngine notLoaded(boolean failed) { + PropertyListCloudEngine engine = mock(PropertyListCloudEngine.class); + when(engine.getElementDataKey()).thenReturn("device"); + when(engine.hasLoadedProperties()).thenReturn(false); + when(engine.propertyListRequestFailed()).thenReturn(failed); + when(engine.getProperties()).thenReturn( + Collections.emptyList()); + return engine; + } + + @SuppressWarnings("unchecked") + private static AspectEngine onPremise(Class dataType) { + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + when(engine.getElementDataKey()).thenReturn("device"); + when(engine.getDataSourceTier()).thenReturn("Lite"); + when(engine.hasLoadedProperties()).thenReturn(true); + if (dataType != null) { + doReturn(new TypedKeyDefault<>("device", dataType)) + .when(engine).getTypedDataKey(); + } + return engine; + } + + private static void listed( + AspectEngine engine, + String tier, + boolean available) { + listed(engine, tier, available, "HardwareVendor"); + } + + private static void listed( + AspectEngine engine, + String tier, + boolean available, + String name) { + AspectPropertyMetaData property = new AspectPropertyMetaDataDefault( + name, + engine, + "", + String.class, + Arrays.asList(tier), + available); + doReturn(Arrays.asList(property)).when(engine).getProperties(); + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyServiceTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyServiceTests.java index 5c81ab7f8..f63f33e83 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyServiceTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/MissingPropertyServiceTests.java @@ -28,6 +28,7 @@ import fiftyone.pipeline.engines.data.AspectPropertyMetaDataDefault; import fiftyone.pipeline.engines.flowelements.AspectEngine; import fiftyone.pipeline.engines.flowelements.CloudAspectEngine; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; import org.junit.Before; import org.junit.Test; @@ -51,13 +52,14 @@ public void Initialise() { /** * Check that an "upgrade required" reason is returned when a property is - * available, but in another data tier. + * available, but in another data tier, which only an on-premise engine + * has. */ @SuppressWarnings("unchecked") @Test public void MissingPropertyService_GetReason_Upgrade() { // Arrange - AspectEngine engine = mock(AspectEngine.class); + AspectEngine engine = mock(OnPremiseAspectEngine.class); when(engine.getDataSourceTier()).thenReturn("lite"); when(engine.getElementDataKey()).thenReturn("testElement"); configureProperty(engine); @@ -156,6 +158,7 @@ public void MissingPropertyService_GetReason_ProductNotInResource() { // Arrange CloudAspectEngine engine = mock(CloudAspectEngine.class); when(engine.getElementDataKey()).thenReturn("testElement"); + when(engine.hasLoadedProperties()).thenReturn(true); when(engine.getProperties()).thenReturn(Collections.emptyList()); // Act @@ -188,6 +191,7 @@ public void MissingPropertyService_GetReason_PropertyNotInResource() { // Arrange CloudAspectEngine engine = mock(CloudAspectEngine.class); when(engine.getElementDataKey()).thenReturn("testElement"); + when(engine.hasLoadedProperties()).thenReturn(true); configureProperty(engine); // Act diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/testhelpers/flowelements/ConfiguredEngineBuilder.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/testhelpers/flowelements/ConfiguredEngineBuilder.java new file mode 100644 index 000000000..6916a1c5f --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/testhelpers/flowelements/ConfiguredEngineBuilder.java @@ -0,0 +1,78 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.testhelpers.flowelements; + +import fiftyone.pipeline.annotations.ElementBuilder; +import org.slf4j.ILoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * A builder for an {@link EmptyEngine} that records the options a + * configuration file gave it, so a test can check how they were read. + */ +@ElementBuilder +public class ConfiguredEngineBuilder extends EmptyEngineBuilder { + + /** + * An option whose builder method takes only an enumeration. + */ + public enum Speed { + SLOW, + FAST + } + + /** + * The properties the last engine was built with. + */ + public static volatile List lastProperties = null; + + /** + * The speed the last engine was built with. + */ + public static volatile Speed lastSpeed = null; + + private Speed speed = Speed.SLOW; + + public ConfiguredEngineBuilder(ILoggerFactory loggerFactory) { + super(loggerFactory); + } + + /** + * Set the speed. There is no overload that takes a string. + * @param speed the speed + * @return this builder + */ + public ConfiguredEngineBuilder setSpeed(Speed speed) { + this.speed = speed; + return this; + } + + @Override + public EmptyEngine build() throws Exception { + lastProperties = new ArrayList<>(getProperties()); + lastSpeed = speed; + return super.build(); + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/testhelpers/flowelements/EmptyEngine.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/testhelpers/flowelements/EmptyEngine.java index 0230799c7..39b2532dd 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/testhelpers/flowelements/EmptyEngine.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/testhelpers/flowelements/EmptyEngine.java @@ -37,11 +37,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; public class EmptyEngine extends AspectEngineBase { private long processCost = 0; + private final AtomicInteger processCount = new AtomicInteger(); private Exception exception = null; private List properties = null; private EvidenceKeyFilterWhitelist evidenceWhitelist = @@ -70,6 +72,14 @@ public void setProcessCost(long milliseconds) { processCost = milliseconds; } + /** + * @return the number of times the engine has processed, which does not + * count results taken from the cache + */ + public int getProcessCount() { + return processCount.get(); + } + public void setException(Exception exception) { this.exception = exception; } @@ -101,6 +111,7 @@ public String getDataSourceTier() { @Override protected void processEngine(FlowData data, EmptyEngineData aspectData) throws Exception { + processCount.incrementAndGet(); if (exception != null) { throw exception; } diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/testhelpers/flowelements/TextFileEngine.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/testhelpers/flowelements/TextFileEngine.java new file mode 100644 index 000000000..2018f247d --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/testhelpers/flowelements/TextFileEngine.java @@ -0,0 +1,140 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.testhelpers.flowelements; + +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.factories.ElementDataFactory; +import fiftyone.pipeline.core.typed.TypedKey; +import fiftyone.pipeline.core.typed.TypedKeyDefault; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngineBase; +import fiftyone.pipeline.engines.testhelpers.data.EmptyEngineData; +import org.slf4j.Logger; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +/** + * An on-premise engine whose data file is a text file, which it reads when it + * is built and again whenever its data is refreshed. + */ +public class TextFileEngine + extends OnPremiseAspectEngineBase { + + private final String dataFilePath; + + private volatile String text; + + @SuppressWarnings("this-escape") + private final TypedKey typedKey = + new TypedKeyDefault<>(getElementDataKey(), EmptyEngineData.class); + + public TextFileEngine( + String dataFilePath, + Logger logger, + ElementDataFactory aspectDataFactory, + String tempDataDirPath) throws IOException { + super(logger, aspectDataFactory, tempDataDirPath); + this.dataFilePath = dataFilePath; + read(); + } + + private void read() throws IOException { + text = new String( + Files.readAllBytes(Paths.get(dataFilePath)), + StandardCharsets.UTF_8); + } + + /** + * @return the text the data file held when it was last read + */ + public String getText() { + return text; + } + + @Override + public Date getDataFilePublishedDate(String dataFileIdentifier) { + return null; + } + + @Override + public Date getDataFileUpdateAvailableTime(String dataFileIdentifier) { + return null; + } + + @Override + public void refreshData(String dataFileIdentifier) { + try { + read(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + @Override + public void refreshData(String dataFileIdentifier, byte[] data) { + text = new String(data, StandardCharsets.UTF_8); + } + + @Override + protected void processEngine(FlowData data, EmptyEngineData aspectData) { + aspectData.setValueOne(text.length()); + } + + @Override + public String getElementDataKey() { + return "text-file"; + } + + @Override + public TypedKey getTypedDataKey() { + return typedKey; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist( + Collections.emptyList()); + } + + @Override + public List getProperties() { + return Collections.emptyList(); + } + + @Override + public String getDataSourceTier() { + return "free"; + } + + @Override + protected void unmanagedResourcesCleanup() { + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/testhelpers/flowelements/TextFileEngineBuilder.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/testhelpers/flowelements/TextFileEngineBuilder.java new file mode 100644 index 000000000..3eb6752fc --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/testhelpers/flowelements/TextFileEngineBuilder.java @@ -0,0 +1,73 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.testhelpers.flowelements; + +import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.configuration.DataFileConfiguration; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.flowelements.AspectEngine; +import fiftyone.pipeline.engines.flowelements.SingleFileAspectEngineBuilderBase; +import fiftyone.pipeline.engines.services.DataUpdateService; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import fiftyone.pipeline.engines.testhelpers.data.EmptyEngineData; +import org.slf4j.ILoggerFactory; + +import java.util.List; + +/** + * Builds a {@link TextFileEngine} from one data file. + */ +public class TextFileEngineBuilder extends SingleFileAspectEngineBuilderBase< + TextFileEngineBuilder, + TextFileEngine> { + + public TextFileEngineBuilder( + ILoggerFactory loggerFactory, + DataUpdateService dataUpdateService) { + super(loggerFactory, dataUpdateService); + } + + @Override + @SuppressWarnings("unchecked") + protected TextFileEngine newEngine(List properties) + throws Exception { + DataFileConfiguration config = dataFileConfigs.get(0); + return new TextFileEngine( + config.getDataFilePath(), + loggerFactory.getLogger(TextFileEngine.class.getName()), + (flowData, element) -> new EmptyEngineData( + loggerFactory.getLogger(EmptyEngineData.class.getName()), + flowData, + (AspectEngine) element, + MissingPropertyServiceDefault.getInstance()), + tempDir); + } + + @Override + public TextFileEngineBuilder setPerformanceProfile( + Constants.PerformanceProfiles profile) { + return this; + } +} diff --git a/pipeline/pipeline.javascriptbuilder/pom.xml b/pipeline/pipeline.javascriptbuilder/pom.xml index 7a7a03bf8..73a9e1f19 100644 --- a/pipeline/pipeline.javascriptbuilder/pom.xml +++ b/pipeline/pipeline.javascriptbuilder/pom.xml @@ -53,6 +53,21 @@ compiler ${mustache-compiler.version} + + + net.logicsquad + minifier + ${minifier.version} + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.graalvm.js js-scriptengine diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderDataInternal.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderDataInternal.java index 54daa4c96..0ca38907e 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderDataInternal.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderDataInternal.java @@ -40,6 +40,15 @@ public JavaScriptBuilderDataInternal(Logger logger, FlowData flowData) { super(logger, flowData); } + /** + * Holds only its values, so it is reused through its Flow Element's + * pool once the Flow Data that holds it is closed. + */ + @Override + protected boolean isPoolable() { + return true; + } + @Override public String getJavaScript() { return getAs("javascript", String.class); diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java index 509fc1b0b..3f3c6ee1a 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java @@ -38,13 +38,16 @@ import fiftyone.pipeline.javascriptbuilder.data.JavaScriptBuilderData; import fiftyone.pipeline.javascriptbuilder.templates.JavaScriptResource; import fiftyone.pipeline.jsonbuilder.data.JsonBuilderData; +import net.logicsquad.minifier.js.JSMinifier; import org.json.JSONObject; import org.slf4j.Logger; import java.io.*; import java.net.URLEncoder; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.regex.Matcher; import java.util.regex.Pattern; import static fiftyone.pipeline.core.Constants.*; @@ -69,8 +72,50 @@ public class JavaScriptBuilderElement protected String contextRoot; protected final String objName; protected final boolean enableCookies; + protected final boolean minify; private final Mustache mustache; + /** + * The minified forms of the template, one for each combination of its + * sections, with a placeholder where each value goes. Minifying is done + * once for each form and not for each request, so minification costs + * almost nothing per request, and the values the script carries are + * never passed through the minifier, so it cannot change them. + */ + private final ConcurrentHashMap minifiedForms = + new ConcurrentHashMap<>(); + + /** + * Set once minification has failed and been logged, after which the + * script is not minified. + */ + private final AtomicBoolean minifyFailed = new AtomicBoolean(false); + + /** + * The text each value is given when the template is minified. Each is + * an identifier, so the minifier leaves it as it is, whether it is in + * code or in a string. + */ + private static final String PLACEHOLDER_PREFIX = + "fiftyoneDegreesMinifyPlaceholder"; + + /** + * The values of the template that differ for each request, in the + * order of {@link #PLACEHOLDER_NAMES}. + */ + private static final String[] VALUE_KEYS = { + "_objName", "_jsonObject", "_sessionId", "_sequence", "_url", + "_parameters"}; + + /** + * The placeholder for each of {@link #VALUE_KEYS}. + */ + private static final String[] PLACEHOLDER_NAMES = { + "ObjName", "Json", "SessionId", "Sequence", "Url", "Parameters"}; + + private static final Pattern PLACEHOLDER_PATTERN = Pattern.compile( + PLACEHOLDER_PREFIX + "(" + String.join("|", PLACEHOLDER_NAMES) + ")"); + /** * Set once a callback URL that is not valid has been logged, because a * misconfigured element would otherwise log on every request. @@ -153,8 +198,11 @@ public static boolean isValidObjectName(String name) { * @param host The host that the client JavaScript should query for updates. * If null or blank then the host from the request will be used * @param protocol The protocol (HTTP or HTTPS) that the client JavaScript - * will use when querying for updates. If null or blank + * will use when querying for updates. If null or empty * then the protocol from the request will be used + * @throws PipelineConfigurationException if protocol is not null or empty + * and is neither http nor https in any case of its ASCII letters, or if + * objName is not a valid JavaScript identifier */ public JavaScriptBuilderElement( Logger logger, @@ -182,11 +230,13 @@ public JavaScriptBuilderElement( * @param host The host that the client JavaScript should query for updates. * If null or blank then the host from the request will be used * @param protocol The protocol (HTTP or HTTPS) that the client JavaScript - * will use when querying for updates. If null or blank + * will use when querying for updates. If null or empty * then the protocol from the request will be used * @param contextRoot The <context-root> setting from the web.xml. * This is needed when creating the callback URL. - * @throws PipelineConfigurationException if objName is not null and is + * @throws PipelineConfigurationException if protocol is not null or empty + * and is neither http nor https in any case of its ASCII letters, white + * space included, or if objName is not null and is * not a valid JavaScript identifier, see {@link #isValidObjectName}. * A null name means the default name is used, as no name was * configured, and an empty name is refused @@ -200,6 +250,48 @@ public JavaScriptBuilderElement( String host, String protocol, String contextRoot) { + this(logger, elementDataFactory, endpoint, objName, enableCookies, + host, protocol, contextRoot, true); + } + //! [constructor] + + //! [constructor] + /** + * Constructor. + * @param logger The logger. + * @param elementDataFactory The element data factory. + * @param endpoint Set the endpoint which will be queried on the host. + * e.g /api/v4/json + * @param objName The default name of the object instantiated by the client + * JavaScript. + * @param enableCookies Set whether the client JavaScript stored results of + * client side processing in cookies. + * @param host The host that the client JavaScript should query for updates. + * If null or blank then the host from the request will be used + * @param protocol The protocol (HTTP or HTTPS) that the client JavaScript + * will use when querying for updates. If null or empty + * then the protocol from the request will be used + * @param contextRoot The <context-root> setting from the web.xml. + * This is needed when creating the callback URL. + * @param minify True to minify the JavaScript that is produced. The other + * constructors minify it. + * @throws PipelineConfigurationException if protocol is not null or empty + * and is neither http nor https in any case of its ASCII letters, white + * space included, or if objName is not null and is + * not a valid JavaScript identifier, see {@link #isValidObjectName}. + * A null name means the default name is used, as no name was + * configured, and an empty name is refused + */ + public JavaScriptBuilderElement( + Logger logger, + ElementDataFactory elementDataFactory, + String endpoint, + String objName, + boolean enableCookies, + String host, + String protocol, + String contextRoot, + boolean minify) { super(logger, elementDataFactory); MustacheFactory mf = new DefaultMustacheFactory(); @@ -209,6 +301,13 @@ public JavaScriptBuilderElement( this.host = host; this.endpoint = endpoint; + if (protocol != null && + protocol.isEmpty() == false && + CallbackUrl.isScheme(protocol) == false) { + throw new PipelineConfigurationException( + "Invalid protocol in configuration (" + protocol + + "), must be 'http' or 'https'"); + } this.protocol = protocol; if (objName == null) { this.objName = Constants.DEFAULT_OBJECT_NAME; @@ -220,6 +319,7 @@ public JavaScriptBuilderElement( } this.enableCookies = enableCookies; this.contextRoot = contextRoot; + this.minify = minify; } //! [constructor] @@ -337,7 +437,7 @@ private boolean getSupportsPromises(FlowData data) { AspectPropertyValue supportsPromisesValue = data.getAs("Promise", AspectPropertyValue.class); supportsPromises = supportsPromisesValue.hasValue() && - supportsPromisesValue.getValue() == "Full"; + "Full".equals(supportsPromisesValue.getValue()); } catch (PipelineDataException | PropertyMissingException e) { supportsPromises = false; } @@ -395,9 +495,16 @@ private Map getParameters(FlowData data) throws UnsupportedEncod .getEvidence() .asKeyMap(); + // The prefix with its separator, so that a key such as "querystring" + // is not taken for a query parameter. + String queryPrefix = EVIDENCE_QUERY_PREFIX + EVIDENCE_SEPERATOR; for(Map.Entry entry : queryEvidence.entrySet()){ - if(entry.getKey().startsWith(EVIDENCE_QUERY_PREFIX) && - EXCLUDED_PARAMETERS.contains(entry.getKey()) == false){ + // Evidence keys ignore case, so the prefix and the excluded + // keys are matched whatever the case the key was added in. + if(entry.getKey().regionMatches(true, 0, + queryPrefix, 0, queryPrefix.length()) && + EXCLUDED_PARAMETERS.contains( + entry.getKey().toLowerCase(Locale.ROOT)) == false){ String key = entry.getKey().substring(entry.getKey().indexOf(EVIDENCE_SEPERATOR) + 1); key = URLEncoder.encode(key, "UTF-8"); String value = URLEncoder.encode(entry.getValue().toString(), "UTF-8"); @@ -607,13 +714,88 @@ private void buildJavaScript( cookies, updateEnabled, hasDelayedProperties); - - StringWriter stringWriter = new StringWriter(); - mustache.execute(stringWriter, javaScriptObj.asMap()); - String content = stringWriter.toString(); + Map values = javaScriptObj.asMap(); + String content = null; + if (minify) { + content = renderMinified(values); + } + if (content == null) { + content = render(values); + } elementData.setJavaScript(content); } + + /** + * Renders the template as written. + */ + private String render(Map values) { + StringWriter stringWriter = new StringWriter(); + mustache.execute(stringWriter, values); + return stringWriter.toString(); + } + + /** + * Renders the minified template. The minified form of the template for + * the request's sections is made once and kept, and the request's values + * are put in place of its placeholders. + * @return the minified script, or null if minification failed + */ + private String renderMinified(Map values) { + if (minifyFailed.get()) { + return null; + } + // The form depends only on the template's sections, being every + // value that is not one of the placeholders. The URL is a value, and + // whether it is present is the section _updateEnabled. + int key = 0; + for (String section : new String[] { + "_supportsPromises", "_enableCookies", "_updateEnabled", + "_hasDelayedProperties"}) { + key = (key << 1) | (Boolean.TRUE.equals(values.get(section)) ? 1 : 0); + } + String form = minifiedForms.computeIfAbsent(key, k -> minifyForm(values)); + if (form.isEmpty()) { + return null; + } + + Matcher matcher = PLACEHOLDER_PATTERN.matcher(form); + StringBuffer script = new StringBuffer(form.length() + + String.valueOf(values.get("_jsonObject")).length()); + while (matcher.find()) { + String valueKey = VALUE_KEYS[ + Arrays.asList(PLACEHOLDER_NAMES).indexOf(matcher.group(1))]; + matcher.appendReplacement(script, + Matcher.quoteReplacement(String.valueOf(values.get(valueKey)))); + } + matcher.appendTail(script); + return script.toString(); + } + + /** + * Renders the template with the given sections and a placeholder for + * each value, and minifies it. + * @return the minified form, or an empty string if minification failed + */ + private String minifyForm(Map values) { + Map placeholders = new HashMap<>(values); + for (int i = 0; i < VALUE_KEYS.length; i++) { + placeholders.put(VALUE_KEYS[i], + PLACEHOLDER_PREFIX + PLACEHOLDER_NAMES[i]); + } + try { + StringWriter minified = new StringWriter(); + new JSMinifier(new StringReader(render(placeholders))) + .minify(minified); + return minified.toString(); + } catch (Exception e) { + if (minifyFailed.compareAndSet(false, true)) { + logger.error("The JavaScript could not be minified, so it " + + "is sent as it is written.", e); + } + return ""; + } + } } //! [class] diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java index 47a449373..89d3bcb8a 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java @@ -40,6 +40,10 @@ @ElementBuilder public class JavaScriptBuilderElementBuilder { public static final boolean ENABLE_COOKIES = true; + /** + * Whether the script is minified when nothing else is configured. + */ + public static final boolean MINIFY = true; private final ILoggerFactory loggerFactory; private final Logger logger; @@ -49,6 +53,7 @@ public class JavaScriptBuilderElementBuilder { protected String contextRoot = ""; protected String objName = Constants.DEFAULT_OBJECT_NAME; private boolean enableCookies = ENABLE_COOKIES; + private boolean minify = MINIFY; /** * Construct a new instance. @@ -171,6 +176,22 @@ public JavaScriptBuilderElementBuilder setEnableCookies( return this; } + /** + * Set whether the JavaScript that is produced is minified. Minifying + * takes out the template's comments and white space, so the script is + * smaller to send. The values the script carries, such as the JSON, are + * not changed. + *

    + * Default is true + * @param minify true to minify the JavaScript + * @return this builder + */ + @DefaultValue(booleanValue = MINIFY) + public JavaScriptBuilderElementBuilder setMinify(boolean minify) { + this.minify = minify; + return this; + } + /** * Build the {@link JavaScriptBuilderElement} * @return new {@link JavaScriptBuilderElement} instance @@ -194,7 +215,8 @@ public JavaScriptBuilderData create( enableCookies, host, protocol, - contextRoot); - } + contextRoot, + minify); + } } //! [class] diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderElementBuilderTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderElementBuilderTests.java index 55d0daaa1..8825c30ad 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderElementBuilderTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderElementBuilderTests.java @@ -24,12 +24,15 @@ import fiftyone.common.testhelpers.TestLoggerFactory; import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.LoggerFactory; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -108,4 +111,47 @@ public void JavaScriptBuilderElement_Builder_SetDefaultProtocol_ValidProtocol( loggerFactory.loggers.forEach(testLogger -> assertTrue(testLogger.warningsLogged.size() == 0)); } + + /** + * A protocol given to the element's constructor that is neither http nor + * https, including one made only of white space, is refused when the + * element is made, as one given to the builder is. + * @param protocol + */ + @ParameterizedTest + @ValueSource(strings = {"htp", "ftp", " ", "\t", "https:"}) + public void JavaScriptBuilderElement_Constructor_InvalidProtocol_Refused( + String protocol) { + assertThrows( + PipelineConfigurationException.class, + () -> new JavaScriptBuilderElement( + loggerFactory.getLogger("test"), + null, + "", + null, + false, + "", + protocol)); + } + + /** + * A null or empty protocol means none is configured, so the protocol + * comes from Evidence, and http or https in any case of its ASCII + * letters is accepted. + * @param protocol + */ + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = {"http", "https", "HTTPS", "Http"}) + public void JavaScriptBuilderElement_Constructor_ValidOrNoProtocol_Accepted( + String protocol) { + assertDoesNotThrow(() -> new JavaScriptBuilderElement( + loggerFactory.getLogger("test"), + null, + "", + null, + false, + "", + protocol)); + } } diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderPromiseTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderPromiseTests.java new file mode 100644 index 000000000..11e2a3dac --- /dev/null +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderPromiseTests.java @@ -0,0 +1,145 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.javascriptbuilder; + +import fiftyone.common.testhelpers.TestLoggerFactory; +import fiftyone.pipeline.core.data.Evidence; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.TryGetResult; +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; +import fiftyone.pipeline.engines.fiftyone.data.SequenceData; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement; +import fiftyone.pipeline.javascriptbuilder.data.JavaScriptBuilderData; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; +import fiftyone.pipeline.jsonbuilder.data.JsonBuilderData; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.mockito.stubbing.Answer; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static fiftyone.pipeline.core.Constants.EVIDENCE_PROTOCOL; +import static fiftyone.pipeline.javascriptbuilder.Constants.EVIDENCE_HOST_KEY; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The script takes its promise form when the Promise Property is "Full", + * however the text "Full" was made. A value read from a data file is a + * String made at run time, not the literal in the code that compares it. + */ +public class JavaScriptBuilderPromiseTests { + + private JavaScriptBuilderData result; + + /** + * Renders the script for a request whose Promise Property is the value + * given. + */ + @SuppressWarnings("unchecked") + private String scriptWithPromise(String promise) throws Exception { + ILoggerFactory internalLogger = mock(ILoggerFactory.class); + when(internalLogger.getLogger(anyString())) + .thenReturn(mock(Logger.class)); + TestLoggerFactory loggerFactory = new TestLoggerFactory(internalLogger); + FlowData flowData = mock(FlowData.class); + + JSONObject json = new JSONObject() + .put("device", new JSONObject().put("ismobile", true)); + doAnswer((Answer) invocation -> { + JsonBuilderData jsonData = mock(JsonBuilderData.class); + when(jsonData.getJson()).thenReturn(json.toString(0)); + return jsonData; + }).when(flowData).get(JsonBuilderData.class); + + Map evidence = new HashMap<>(); + evidence.put(EVIDENCE_HOST_KEY, "localhost"); + evidence.put(EVIDENCE_PROTOCOL, "https"); + Evidence evidenceObject = mock(Evidence.class); + when(evidenceObject.asKeyMap()).thenReturn(evidence); + when(flowData.getEvidence()).thenReturn(evidenceObject); + when(flowData.tryGetEvidence(anyString(), any(Class.class))) + .thenAnswer(invocation -> { + TryGetResult found = new TryGetResult<>(); + if (evidence.containsKey((String) invocation.getArgument(0))) { + found.setValue(evidence.get( + (String) invocation.getArgument(0))); + } + return found; + }); + + ElementData sequenceData = mock(ElementData.class); + when(sequenceData.get(SequenceData.SESSION_ID_PROPERTY_NAME)) + .thenReturn("abcdefg-hijklmn"); + when(sequenceData.get(SequenceData.SEQUENCE_PROPERTY_NAME)) + .thenReturn(1); + when(flowData.getDataKeys()).thenReturn(Collections.singletonList( + SequenceElement.DEFAULT_ELEMENT_DATA_KEY)); + when(flowData.get(SequenceElement.DEFAULT_ELEMENT_DATA_KEY)) + .thenReturn(sequenceData); + + when(flowData.getAs("Promise", AspectPropertyValue.class)) + .thenReturn(new AspectPropertyValueDefault<>(promise)); + doAnswer((Answer) invocation -> { + FlowElement.DataFactory factory = + invocation.getArgument(1); + result = factory.create(flowData); + return result; + }).when(flowData).getOrAdd( + anyString(), any(FlowElement.DataFactory.class)); + + new JavaScriptBuilderElementBuilder(loggerFactory) + .build() + .process(flowData); + return result.getJavaScript(); + } + + /** + * "Full" made at run time gives the same script as the literal "Full", + * and both differ from the script for "None", so the comparison is of + * the text and the test can see the promise form. + */ + @Test + public void PromiseFull_MadeAtRunTime_GivesThePromiseForm() + throws Exception { + String literal = scriptWithPromise("Full"); + String madeAtRunTime = scriptWithPromise( + new StringBuilder("Fu").append("ll").toString()); + assertNotEquals(scriptWithPromise("None"), literal, + "The script for Full is the same as for None."); + assertEquals(literal, madeAtRunTime, + "Full made at run time did not give the promise form."); + } +} diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java index 7ec0c95e1..05fc697cc 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java @@ -256,8 +256,10 @@ public void JavaScriptBuilderElement_DelayExecution() throws Exception { public void JavaScriptBuilderElement_VerifySession() throws Exception { configureMocks(); + // Not minified, so the declarations read as the template writes them. javaScriptBuilderElement = new JavaScriptBuilderElementBuilder(loggerFactory) + .setMinify(false) .build(); javaScriptBuilderElement.process(flowData); @@ -326,8 +328,10 @@ public void JavaScriptBuilderElement_ParametersExcludeSessionAndSequence() throws Exception { configureMocks(); + // Not minified, so the script is read line by line as written. javaScriptBuilderElement = new JavaScriptBuilderElementBuilder(loggerFactory) + .setMinify(false) .build(); javaScriptBuilderElement.process(flowData); String javaScript = result.getJavaScript(); @@ -378,9 +382,11 @@ public void JavaScriptBuilderElement_NoParameters() throws Exception { evidence.remove("query.latitude"); evidence.remove("query.longitude"); + // Not minified, so the script is read line by line as written. javaScriptBuilderElement = new JavaScriptBuilderElementBuilder(loggerFactory) .setEndpoint("/json") + .setMinify(false) .build(); javaScriptBuilderElement.process(flowData); diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptMinifyPerformanceTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptMinifyPerformanceTests.java new file mode 100644 index 000000000..8ef437fe0 --- /dev/null +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptMinifyPerformanceTests.java @@ -0,0 +1,144 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.javascriptbuilder; + +import fiftyone.common.testhelpers.TestLoggerFactory; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Measures what minification costs, as the JavaScript Builder specification + * asks of the feature, because a slow minification library once caused a + * customer problems. + *

    + * The element with minification on and the element with it off render in + * turn, many times, so a busy machine slows both alike, and the test fails + * only when minifying makes rendering much slower. The times are printed so + * a change in the cost can be seen. + * @see Specification + */ +public class JavaScriptMinifyPerformanceTests { + + /** + * Renders of each element before measuring, so the first render, which + * minifies the template's form, and the JIT compiler's work are not + * measured. + */ + private static final int WARM_UP = 200; + + /** + * Renders of each element that are measured. + */ + private static final int MEASURED = 2000; + + /** + * How many times slower a minified render may be than one not + * minified. Minifying is done once for each form of the template and + * not for each request, so a minified render costs about the same as + * one that is not minified, and a minifier run on every request is many + * times slower. The margin is wide so that timing on a busy machine + * does not decide the result. + */ + private static final double MAX_RATIO = 2.0; + + private final TestLoggerFactory loggerFactory = new TestLoggerFactory(null); + + @Test + public void Minify_PerformanceImpact_Small() throws Exception { + JavaScriptBuilderElement minified = element(true); + JavaScriptBuilderElement asWritten = element(false); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(new SequenceElementBuilder(loggerFactory).build()) + .addFlowElement(new JsonBuilderElementBuilder(loggerFactory).build()) + // Adds the JavaScript Builder's Element Data while the Flow Data + // is processed, as it is read only afterwards. Both elements + // below write that one instance. + .addFlowElement(minified) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.user-agent", "iPhone"); + data.process(); + + long first = System.nanoTime(); + minified.process(data); + long firstMinified = System.nanoTime() - first; + + for (int i = 0; i < WARM_UP; i++) { + minified.process(data); + asWritten.process(data); + } + + long minifiedTime = 0; + long asWrittenTime = 0; + for (int i = 0; i < MEASURED; i++) { + // Each goes first in turn, so neither always follows the + // other. + if (i % 2 == 0) { + minifiedTime += time(minified, data); + asWrittenTime += time(asWritten, data); + } else { + asWrittenTime += time(asWritten, data); + minifiedTime += time(minified, data); + } + } + + double ratio = (double) minifiedTime / asWrittenTime; + System.out.printf( + "JavaScript render: first minified %.3f ms, then %.4f ms " + + "minified and %.4f ms not minified on average over %d, " + + "ratio %.2f%n", + firstMinified / 1e6, + minifiedTime / 1e6 / MEASURED, + asWrittenTime / 1e6 / MEASURED, + MEASURED, + ratio); + assertTrue(ratio <= MAX_RATIO, String.format( + "A minified render took %.2f times as long as one not " + + "minified, more than the %.1f allowed.", ratio, MAX_RATIO)); + } + } + + private JavaScriptBuilderElement element(boolean minify) { + return new JavaScriptBuilderElementBuilder(loggerFactory) + .setEndpoint("/json") + .setHost("example.com") + .setProtocol("https") + .setMinify(minify) + .build(); + } + + private static long time(JavaScriptBuilderElement element, FlowData data) + throws Exception { + long start = System.nanoTime(); + element.process(data); + return System.nanoTime() - start; + } +} diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptMinifyTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptMinifyTests.java new file mode 100644 index 000000000..cc21c6455 --- /dev/null +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptMinifyTests.java @@ -0,0 +1,412 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.javascriptbuilder; + +import fiftyone.common.testhelpers.TestLoggerFactory; +import fiftyone.pipeline.core.data.ElementDataBase; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.ElementPropertyMetaDataDefault; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.types.JavaScript; +import fiftyone.pipeline.core.flowelements.FlowElementBase; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.PolyglotException; +import org.graalvm.polyglot.Source; +import org.graalvm.polyglot.Value; +import org.json.JSONObject; +import org.json.JSONTokener; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * The script is minified, or not, as configured, and minification is on + * unless it is turned off. See the Process section and the SetMinify + * configuration option of the JavaScript Builder specification. + * @see Specification + */ +public class JavaScriptMinifyTests { + + /** + * A comment in the template, which a minified script does not carry. + */ + private static final String TEMPLATE_COMMENT = + "// Log any errors returned in the JSON object."; + + /** + * A value that looks like code a minifier would change, and holds + * quotes and a line break, so it shows whether minification has changed + * the data the script carries. + */ + private static final String VALUE = + "a /* not a comment */ b // nor this\n\t'c' \"d\" {{_objName}}"; + + private static final String BROWSER = + "var window = globalThis;\n" + + "var console = { log: function() {}, warn: function() {},\n" + + " error: function() {} };\n" + + "var document = { cookie: '' };\n" + + "var sessionStorage = (function() {\n" + + " var data = {};\n" + + " return {\n" + + " get length() { return Object.keys(data).length; },\n" + + " key: function(i) {\n" + + " var k = Object.keys(data)[i];\n" + + " return k === undefined ? null : k; },\n" + + " getItem: function(k) {\n" + + " return k in data ? data[k] : null; },\n" + + " setItem: function(k, v) { data[k] = String(v); },\n" + + " removeItem: function(k) { delete data[k]; }\n" + + " };\n" + + "})();\n"; + + private final TestLoggerFactory loggerFactory = new TestLoggerFactory(null); + + /** + * Renders the script for one request. + * @param minify null to leave the builder's default, otherwise the + * value given to setMinify + * @param endpoint the endpoint, or an empty string for no callback + * @param cookies the value given to setEnableCookies + */ + private String render(Boolean minify, String endpoint, boolean cookies) + throws Exception { + return render(minify, endpoint, cookies, false, false); + } + + /** + * Renders the script for one request, choosing every setting the + * minified forms are held by. + * @param minify null to leave the builder's default, otherwise the + * value given to setMinify + * @param endpoint the endpoint, or an empty string for no callback + * @param cookies the value given to setEnableCookies + * @param promises true if the request's browser supports promises + * @param delayed true if the JSON holds a delayed execution property + */ + private String render( + Boolean minify, + String endpoint, + boolean cookies, + boolean promises, + boolean delayed) throws Exception { + JavaScriptBuilderElementBuilder builder = + new JavaScriptBuilderElementBuilder(loggerFactory) + .setEndpoint(endpoint) + .setHost("example.com") + .setProtocol("https") + .setEnableCookies(cookies); + if (minify != null) { + builder.setMinify(minify); + } + JavaScriptBuilderElement jsElement = builder.build(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(new SequenceElementBuilder(loggerFactory).build()) + .addFlowElement(new ValuesElement(promises, delayed)) + .addFlowElement(new JsonBuilderElementBuilder(loggerFactory).build()) + .addFlowElement(jsElement) + .build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.some-value", "x y"); + // The same session for every render, so two renders differ only + // in whether they are minified. + flowData.addEvidence("query.session-id", "abc-123"); + flowData.addEvidence("query.sequence", "1"); + flowData.process(); + return flowData.getFromElement(jsElement).getJavaScript(); + } + } + + /** + * With no setting the script is minified, so it carries none of the + * template's comments and is much shorter than the script made with + * minification turned off. + */ + @Test + public void Minify_Default_ScriptIsMinified() throws Exception { + String minified = render(null, "/json", true); + String asWritten = render(false, "/json", true); + + assertFalse(minified.contains(TEMPLATE_COMMENT), + "The default script still carries the template's comments."); + assertTrue(minified.length() < asWritten.length() * 3 / 4, + "The default script is " + minified.length() + + " characters and the script with minification off is " + + asWritten.length() + "."); + } + + /** + * With minification turned on the script is minified as it is with no + * setting. + */ + @Test + public void Minify_On_ScriptIsMinified() throws Exception { + assertEquals(render(null, "/json", true), render(true, "/json", true)); + } + + /** + * With minification turned off the script is the template as written. + */ + @Test + public void Minify_Off_ScriptIsNotMinified() throws Exception { + assertTrue(render(false, "/json", true).contains(TEMPLATE_COMMENT)); + } + + /** + * Minification changes the code and not what the script carries: the + * JSON, the callback URL, the parameters, the session id, the sequence + * and the object name are the same as in the script made with + * minification turned off. + */ + @Test + public void Minify_On_ValuesUnchanged() throws Exception { + String minified = render(true, "/json", true); + String asWritten = render(false, "/json", true); + + assertTrue(json(asWritten).similar(json(minified)), + "The JSON differs.\n" + json(asWritten).toString(2) + "\n" + + json(minified).toString(2)); + assertEquals(VALUE, json(minified).getJSONObject("values") + .getString("text")); + assertTrue(minified.contains("'https://example.com/json'"), + "The callback URL is missing."); + assertEquals( + parameters(asWritten).toString(), + parameters(minified).toString()); + assertTrue(minified.contains("\"" + sessionId(asWritten) + "\""), + "The session id is missing."); + assertTrue(Pattern.compile("var fod\\s*=\\s*new fiftyoneDegreesManager") + .matcher(minified).find(), "The object is not created."); + } + + /** + * Every choice of the four settings the minified forms are held by, + * being whether there is a callback, whether cookies are enabled, + * whether the browser supports promises and whether the JSON holds a + * delayed execution property. + */ + static Stream everyForm() { + List forms = new ArrayList<>(); + for (String endpoint : new String[] {"/json", ""}) { + for (boolean cookies : new boolean[] {true, false}) { + for (boolean promises : new boolean[] {true, false}) { + for (boolean delayed : new boolean[] {true, false}) { + forms.add(Arguments.of( + endpoint, cookies, promises, delayed)); + } + } + } + } + return forms.stream(); + } + + /** + * Each of the four settings changes the script as written, so the + * sixteen choices are sixteen different scripts and the forms tested + * below are every form the element holds. + */ + @Test + public void Minify_EveryForm_IsADifferentScript() throws Exception { + Set scripts = new HashSet<>(); + Iterable forms = everyForm()::iterator; + for (Arguments form : forms) { + Object[] a = form.get(); + scripts.add(render(false, (String) a[0], (Boolean) a[1], + (Boolean) a[2], (Boolean) a[3])); + } + assertEquals(16, scripts.size(), + "The four settings do not give sixteen different scripts."); + } + + /** + * Every form of the minified script parses. + */ + @ParameterizedTest + @MethodSource("everyForm") + public void Minify_On_EveryFormParses( + String endpoint, + boolean cookies, + boolean promises, + boolean delayed) throws Exception { + String script = render(true, endpoint, cookies, promises, delayed); + try (Context context = context()) { + try { + context.parse(Source.create("js", script)); + } catch (PolyglotException e) { + fail("The minified script does not parse: " + e.getMessage()); + } + } + } + + /** + * The minified script runs and creates the object with the values from + * the JSON. + */ + @Test + public void Minify_On_ScriptRuns() throws Exception { + String script = render(true, "", true); + try (Context context = context()) { + context.eval("js", BROWSER); + context.eval(Source.create("js", script)); + Value fod = context.eval("js", "window").getMember("fod"); + assertFalse(fod == null || fod.isNull(), "fod was not created."); + assertTrue(fod.getMember("complete").canExecute()); + assertEquals(VALUE, + fod.getMember("values").getMember("text").asString()); + } + } + + private static Context context() { + return Context.newBuilder("js") + .option("engine.WarnInterpreterOnly", "false") + .build(); + } + + /** + * The JSON the script assigns to its json variable. + */ + private static JSONObject json(String script) { + Matcher matcher = Pattern.compile("var json\\s*=\\s*").matcher(script); + assertTrue(matcher.find(), "The script holds no JSON."); + return (JSONObject) new JSONTokener(script.substring(matcher.end())) + .nextValue(); + } + + /** + * The parameters object the script's request parameters function + * returns. + */ + private static JSONObject parameters(String script) { + Matcher matcher = Pattern.compile( + "renderedParameters\\s*=\\s*function\\(\\)\\s*\\{\\s*return\\s*") + .matcher(script); + assertTrue(matcher.find(), "The script holds no parameters."); + return (JSONObject) new JSONTokener(script.substring(matcher.end())) + .nextValue(); + } + + private static String sessionId(String script) { + Matcher matcher = Pattern.compile("var sessionId\\s*=\\s*\"([^\"]*)\"") + .matcher(script); + assertTrue(matcher.find(), "The script holds no session id."); + return matcher.group(1); + } + + /** + * An element that adds a value which looks like code, in place of an + * engine. + */ + private static class ValuesElement + extends FlowElementBase { + + private final boolean promises; + + private final boolean delayed; + + /** + * @param promises true to give Promise as "Full", which the + * JavaScript Builder reads as the browser + * supporting promises + * @param delayed true to add a delayed execution property + */ + ValuesElement(boolean promises, boolean delayed) { + super(new TestLoggerFactory(null).getLogger("values"), null); + this.promises = promises; + this.delayed = delayed; + } + + @Override + protected void processInternal(FlowData data) { + ElementDataBase values = new ElementDataBase(logger, data) { }; + values.put("text", VALUE); + if (promises) { + values.put("Promise", + new AspectPropertyValueDefault<>("Full")); + } + if (delayed) { + values.put("delayed", new JavaScript("var delayed = 1;")); + } + data.getOrAdd(getElementDataKey(), d -> values); + } + + @Override + public String getElementDataKey() { + return "values"; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist( + Collections.emptyList()); + } + + @Override + public List getProperties() { + List properties = new ArrayList<>(); + properties.add(new ElementPropertyMetaDataDefault( + "text", this, "test", String.class, true)); + if (promises) { + properties.add(new ElementPropertyMetaDataDefault( + "Promise", this, "test", String.class, true)); + } + if (delayed) { + properties.add(new ElementPropertyMetaDataDefault( + "delayed", this, "test", JavaScript.class, true, null, + true, Collections.emptyList())); + } + return properties; + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + } +} diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java index 712bc6873..119ab8149 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java @@ -316,7 +316,9 @@ public void ObjectName_WindowPropertyVarCanReplace_Allowed(String name) { String script = render(element); - assertTrue(script.contains("var " + name + " = new"), + assertTrue(Pattern.compile( + "var " + Pattern.quote(name) + "\\s*=\\s*new") + .matcher(script).find(), "the script should declare '" + name + "'"); assertEquals(0, warnings().size()); } @@ -356,16 +358,20 @@ public void ObjectName_NotConfigured_DefaultUsed() { } private void assertWorkingObject(String script, String name) { + // The script is minified by default, so the patterns allow for the + // white space around '=' being taken out. assertTrue(Pattern.compile( "var " + Pattern.quote(name) + - " = new fiftyoneDegreesManager\\(\\);") + "\\s*=\\s*new fiftyoneDegreesManager\\(\\);") .matcher(script).find(), "the script should declare var " + name); if (name.equals("fod") == false) { assertFalse(Pattern.compile("var fod\\b").matcher(script).find(), "the script should not declare var fod"); } - assertTrue(script.contains("var sessionKey = \"" + name + "\";"), + assertTrue(Pattern.compile( + "var sessionKey\\s*=\\s*\"" + Pattern.quote(name) + "\";") + .matcher(script).find(), "the storage key should be " + name); assertTrue(script.contains("window[\"" + name + "Evidence\"]"), "the evidence lookup should use " + name); @@ -406,25 +412,27 @@ private List warnings() { } /** - * The script with the line that assigns its request parameters taken - * out. The script sends the page's query evidence, the requested object - * name included, back with its own request. The builder url encodes - * each value and the line holds them as JSON strings, so the requested - * name there is data and not code, and it is left out of the checks - * that the name was not written into the code. The line is found as the - * existing tests find it, by the word and a brace, because the template - * has named the declaration differently over time. + * The script with the object holding its request parameters taken out. + * The script sends the page's query evidence, the requested object name + * included, back with its own request. The builder url encodes each + * value, so no value holds a brace, and the object holds them as JSON + * strings, so the requested name there is data and not code, and it is + * left out of the checks that the name was not written into the code. + * The object is found by the word parameters and a brace, because the + * template has named the declaration differently over time, and not by + * line, because the script is minified by default. */ + private static final Pattern PARAMETERS = Pattern.compile( + "(?i)\\w*parameters\\s*=\\s*" + + "(?:function\\s*\\(\\)\\s*\\{\\s*return\\s*)?\\{[^}]*\\}"); + private static String withoutParameters(String script) { - StringBuilder rest = new StringBuilder(); - for (String line : script.split("\\r?\\n")) { - if (line.toLowerCase().contains("parameters =") && - line.contains("{")) { - continue; - } - rest.append(line).append('\n'); - } - return rest.toString(); + Matcher matcher = PARAMETERS.matcher(script); + assertTrue(matcher.find(), "the script should assign its parameters"); + String rest = matcher.replaceFirst(""); + assertFalse(PARAMETERS.matcher(rest).find(), + "the script should assign its parameters once"); + return rest; } private static int count(String text, String find) { diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java index fc8cb2f61..5540c29ae 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java @@ -55,7 +55,9 @@ /** * Checks the callback URL the rendered script is given. These tests render - * the script only and need no browser. + * the script only and need no browser. The script is not minified, so it is + * read as the template writes it. {@link JavaScriptMinifyTests} checks that + * the minified script carries the same values. */ public class JavaScriptUrlTests { private final TestLoggerFactory loggerFactory; @@ -72,7 +74,8 @@ private String render(String contextRoot) throws Exception { new JavaScriptBuilderElementBuilder(loggerFactory) .setEndpoint("/json") .setHost("example.com") - .setProtocol("https"); + .setProtocol("https") + .setMinify(false); if (contextRoot != null) { builder.setContextRoot(contextRoot); } @@ -139,6 +142,91 @@ public void Url_CarriesNoQueryEvidence() throws Exception { "The parameters object lost the query evidence: " + parameters); } + /** + * Evidence keys ignore case, so query Evidence whose prefix is not in + * lower case reaches the parameters object, and the session id and + * sequence stay out of it whatever their case. + */ + @Test + public void Parameters_QueryPrefixInAnyCase() throws Exception { + JavaScriptBuilderElement jsElement = + new JavaScriptBuilderElementBuilder(loggerFactory) + .setEndpoint("/json") + .setHost("example.com") + .setProtocol("https") + .setMinify(false) + .build(); + String javaScript; + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(new SequenceElementBuilder(loggerFactory).build()) + .addFlowElement(new JsonBuilderElementBuilder(loggerFactory).build()) + .addFlowElement(jsElement) + .build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("Query.Extra", "one"); + flowData.addEvidence("QUERY.Other", "two"); + flowData.addEvidence("Query.Session-Id", "a-session"); + flowData.process(); + javaScript = flowData.getFromElement(jsElement).getJavaScript(); + } + String parameters = null; + for (String line : javaScript.split("\\r?\\n")) { + if (line.toLowerCase().contains("parameters =") + && line.contains("{")) { + parameters = line; + break; + } + } + assertNotNull(parameters, "No parameters object in the script."); + assertTrue(parameters.contains("\"Extra\":\"one\""), + "Query.Extra is not in the parameters: " + parameters); + assertTrue(parameters.contains("\"Other\":\"two\""), + "QUERY.Other is not in the parameters: " + parameters); + assertFalse(parameters.toLowerCase().contains("session-id"), + "The session id is in the parameters: " + parameters); + } + + /** + * Only a key whose prefix is "query" followed by the separator is a + * query parameter, so a key such as "queryx.field" is left out. + */ + @Test + public void Parameters_PrefixWithoutSeparator_IsLeftOut() + throws Exception { + JavaScriptBuilderElement jsElement = + new JavaScriptBuilderElementBuilder(loggerFactory) + .setEndpoint("/json") + .setHost("example.com") + .setProtocol("https") + .setMinify(false) + .build(); + String javaScript; + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(new SequenceElementBuilder(loggerFactory).build()) + .addFlowElement(new JsonBuilderElementBuilder(loggerFactory).build()) + .addFlowElement(jsElement) + .build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.kept", "yes"); + flowData.addEvidence("queryx.field", "not-a-query"); + flowData.process(); + javaScript = flowData.getFromElement(jsElement).getJavaScript(); + } + String parameters = null; + for (String line : javaScript.split("\\r?\\n")) { + if (line.toLowerCase().contains("parameters =") + && line.contains("{")) { + parameters = line; + break; + } + } + assertNotNull(parameters, "No parameters object in the script."); + assertTrue(parameters.contains("\"kept\":\"yes\""), + "query.kept is not in the parameters: " + parameters); + assertFalse(parameters.contains("not-a-query"), + "queryx.field is in the parameters: " + parameters); + } + /** * A context root still sits between the host and the endpoint, and the * URL still carries no query string. @@ -168,6 +256,7 @@ private List renderWithHosts(String... hosts) throws Exception { JavaScriptBuilderElement jsElement = new JavaScriptBuilderElementBuilder(factory) .setEndpoint("/json") + .setMinify(false) .build(); List scripts = new ArrayList<>(); try (Pipeline pipeline = new PipelineBuilder(factory) @@ -270,6 +359,7 @@ public void Url_RefusedProtocol_NoCallbackAndLoggedOnce() throws Exception { JavaScriptBuilderElement jsElement = new JavaScriptBuilderElementBuilder(factory) .setEndpoint("/json") + .setMinify(false) .build(); try (Pipeline pipeline = new PipelineBuilder(factory) .addFlowElement(new SequenceElementBuilder(factory).build()) diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/PooledElementDataTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/PooledElementDataTests.java new file mode 100644 index 000000000..4c00cf7eb --- /dev/null +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/PooledElementDataTests.java @@ -0,0 +1,83 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.javascriptbuilder; + +import fiftyone.common.testhelpers.TestLoggerFactory; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * The JSON Builder and JavaScript Builder Element Data hold only their + * values, so they are pooled. Requests made one after another reuse one + * instance of each. + * @see Specification + */ +public class PooledElementDataTests { + + private static final int REQUESTS = 100; + + private final TestLoggerFactory loggerFactory = + new TestLoggerFactory(null); + + @Test + public void oneThread_OneInstanceOfEach() throws Exception { + Set json = + Collections.newSetFromMap(new IdentityHashMap<>()); + Set javascript = + Collections.newSetFromMap(new IdentityHashMap<>()); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement( + new SequenceElementBuilder(loggerFactory).build()) + .addFlowElement( + new JsonBuilderElementBuilder(loggerFactory).build()) + .addFlowElement( + new JavaScriptBuilderElementBuilder(loggerFactory) + .build()) + .setAutoCloseElements(true) + .build()) { + for (int i = 0; i < REQUESTS; i++) { + try (FlowData data = pipeline.createFlowData()) { + data.process(); + json.add(data.get("json-builder")); + javascript.add(data.get("javascript-builder")); + } + } + } + assertEquals(1, json.size(), "JSON Builder Element Data allocated"); + assertEquals( + 1, + javascript.size(), + "JavaScript Builder Element Data allocated"); + } +} diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java index 06bc9b6b6..e5628a3d6 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java @@ -76,11 +76,14 @@ */ public class SessionIdAndSequenceTests { + // The script is minified by default, so the patterns allow for the + // white space around '=' being taken out and for the rest of the + // script following on the same line. private static final Pattern SESSION_ID_LINE = - Pattern.compile("var sessionId = \"(.*)\";"); + Pattern.compile("var sessionId\\s*=\\s*\"([^\"]*)\";"); private static final Pattern SEQUENCE_LINE = - Pattern.compile("var sequence = (.*);"); + Pattern.compile("var sequence\\s*=\\s*([^;]*);"); private final TestLoggerFactory loggerFactory = new TestLoggerFactory(null); diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/WritesAfterProcessingTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/WritesAfterProcessingTests.java new file mode 100644 index 000000000..3886f3e2c --- /dev/null +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/WritesAfterProcessingTests.java @@ -0,0 +1,87 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.javascriptbuilder; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * The check, for the Flow Element in this module, the + * {@link JavaScriptBuilderElement}, that processes it, then writes to its + * Element Data and expects the error. + * @see Specification + */ +public class WritesAfterProcessingTests { + + private static final ILoggerFactory LOGGER_FACTORY = + LoggerFactory.getILoggerFactory(); + + static void assertWritesFail(ElementData data) { + assertThrows( + IllegalStateException.class, + () -> data.put("after", "value"), + "put"); + assertThrows( + IllegalStateException.class, + () -> data.populateFromMap( + Collections.singletonMap("after", "value")), + "populateFromMap"); + } + + @Test + public void javaScriptBuilder_ElementDataRefusesWrites() + throws Exception { + JavaScriptBuilderElement javaScriptBuilder = + new JavaScriptBuilderElementBuilder(LOGGER_FACTORY).build(); + try (Pipeline pipeline = new PipelineBuilder(LOGGER_FACTORY) + .addFlowElement( + new SequenceElementBuilder(LOGGER_FACTORY).build()) + .addFlowElement( + new JsonBuilderElementBuilder(LOGGER_FACTORY).build()) + .addFlowElement(javaScriptBuilder) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + data.process(); + assertEquals(0, data.getErrors().size(), + String.valueOf(data.getErrors())); + + ElementData own = data.getFromElement(javaScriptBuilder); + assertWritesFail(own); + } + } +} diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/ElementDataKeyFormatTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/ElementDataKeyFormatTests.java new file mode 100644 index 000000000..b9bfbf6e2 --- /dev/null +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/ElementDataKeyFormatTests.java @@ -0,0 +1,90 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.javascriptbuilder.flowelements; + +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SetHeadersElementBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.ShareUsageBuilder; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The string key of each Pipeline API element's data contains only + * alphanumeric characters, full stop or hyphen. + *

    + * Specification: pipeline-specification/features/access-to-results.md line 7 + * at 51Degrees/specifications 706de1f0, "Strings SHOULD only contain + * alphanumeric characters, full stop or hyphen." + */ +public class ElementDataKeyFormatTests { + + private static List> elements() throws Exception { + ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory(); + List> elements = new ArrayList<>(); + elements.add(new JavaScriptBuilderElementBuilder(loggerFactory).build()); + elements.add(new JsonBuilderElementBuilder(loggerFactory).build()); + elements.add(new SequenceElementBuilder(loggerFactory).build()); + elements.add(new SetHeadersElementBuilder(loggerFactory).build()); + elements.add(new ShareUsageBuilder(loggerFactory).build()); + return elements; + } + + @Test + public void keys_OnlyAlphanumericFullStopOrHyphen() throws Exception { + for (FlowElement element : elements()) { + try { + String key = element.getElementDataKey(); + assertTrue(key.matches("[A-Za-z0-9.-]+"), + element.getClass().getSimpleName() + " key '" + key + "'"); + } finally { + element.close(); + } + } + } + + /** + * Specification: access-to-results.md line 10, "Lower-case SHOULD be + * used." + */ + @Test + public void keys_LowerCase() throws Exception { + for (FlowElement element : elements()) { + try { + String key = element.getElementDataKey(); + assertTrue(key.equals(key.toLowerCase(Locale.ROOT)), + element.getClass().getSimpleName() + " key '" + key + "'"); + } finally { + element.close(); + } + } + } +} diff --git a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderDataInternal.java b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderDataInternal.java index 1340b0920..3033bb28d 100644 --- a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderDataInternal.java +++ b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderDataInternal.java @@ -34,6 +34,15 @@ public JsonBuilderDataInternal(Logger logger, FlowData flowData) { super(logger, flowData); } + /** + * Holds only its values, so it is reused through its Flow Element's + * pool once the Flow Data that holds it is closed. + */ + @Override + protected boolean isPoolable() { + return true; + } + @Override public String getJson() { return getAs("json", String.class); diff --git a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java index 72f037049..c1eeab5af 100644 --- a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java +++ b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java @@ -47,6 +47,7 @@ import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; +import org.json.JSONArray; import org.json.JSONObject; import org.slf4j.Logger; @@ -241,12 +242,12 @@ private Map getAllProperties( for (Map.Entry element : data.elementDataAsMap().entrySet()) { if (elementBlacklist.contains(element.getKey()) == false && - allProperties.containsKey(element.getKey().toLowerCase()) == false) { + allProperties.containsKey(element.getKey().toLowerCase(Locale.ROOT)) == false) { Map elementProperties = getValues( - element.getKey().toLowerCase(), + element.getKey().toLowerCase(Locale.ROOT), ((ElementData)element.getValue()).asKeyMap(), config); - allProperties.put(element.getKey().toLowerCase(), elementProperties); + allProperties.put(element.getKey().toLowerCase(Locale.ROOT), elementProperties); } } @@ -279,7 +280,7 @@ private Map getValues( throw new IllegalArgumentException("config"); } - dataPath = dataPath.toLowerCase(); + dataPath = dataPath.toLowerCase(Locale.ROOT); Map values = new HashMap<>(); for (Map.Entry value : sourceData.entrySet()) { Object propertyValue = null; @@ -291,8 +292,8 @@ private Map getValues( propertyValue = aspectProperty.getValue(); } else { - values.put(value.getKey().toLowerCase(), null); - values.put(value.getKey().toLowerCase() + "nullreason", + values.put(value.getKey().toLowerCase(Locale.ROOT), null); + values.put(value.getKey().toLowerCase(Locale.ROOT) + "nullreason", aspectProperty.getNoValueMessage()); } } @@ -302,7 +303,7 @@ private Map getValues( String completeName = dataPath + EVIDENCE_SEPERATOR + - value.getKey().toLowerCase(); + value.getKey().toLowerCase(Locale.ROOT); if (propertyValue != null) { // If the value is a list of complex types then @@ -316,7 +317,7 @@ private Map getValues( List results = new ArrayList<>(); for (Object elementData : elementDatas) { results.add(getValues( - dataPath + "." + value.getKey().toLowerCase(), + dataPath + "." + value.getKey().toLowerCase(Locale.ROOT), ((ElementData) elementData).asKeyMap(), config)); } @@ -329,11 +330,11 @@ else if (propertyValue instanceof List && } // Add this value to the output - values.put(value.getKey().toLowerCase(), propertyValue); + values.put(value.getKey().toLowerCase(Locale.ROOT), propertyValue); // Add 'delayexecution' flag if needed. if (config.delayedExecutionProperties.contains(completeName)) { - values.put(value.getKey().toLowerCase() + "delayexecution", true); + values.put(value.getKey().toLowerCase(Locale.ROOT) + "delayexecution", true); } } // Add evidence properties list if needed. @@ -341,7 +342,7 @@ else if (propertyValue instanceof List && if (config.delayedEvidenceProperties.containsKey(completeName)) { List evidenceProperties = config.delayedEvidenceProperties .get(completeName); - values.put(value.getKey().toLowerCase() + "evidenceproperties", evidenceProperties); + values.put(value.getKey().toLowerCase(Locale.ROOT) + "evidenceproperties", evidenceProperties); } } return values; @@ -392,7 +393,7 @@ private List getJavaScriptProperties( for (Object propertyObject : entry.entrySet()) { Map.Entry property = (Entry)propertyObject; props.add( - element.getKey().toLowerCase() + EVIDENCE_SEPERATOR + property.getKey().toString().toLowerCase()); + element.getKey().toLowerCase(Locale.ROOT) + EVIDENCE_SEPERATOR + property.getKey().toString().toLowerCase(Locale.ROOT)); } } } @@ -422,37 +423,45 @@ private List getJavaScriptProperties( keys = javascriptPropertiesMap.keySet().toArray(keys); for(String key : keys) { if(containsIgnoreCase(props, key) == false) { - javascriptPropertiesMap.remove(key.toLowerCase()); + javascriptPropertiesMap.remove(key.toLowerCase(Locale.ROOT)); } } List javascriptPropertyNames = new ArrayList<>(); for (String name : javascriptPropertiesMap.keySet()) { - javascriptPropertyNames.add(name.toLowerCase()); + javascriptPropertyNames.add(name.toLowerCase(Locale.ROOT)); } return javascriptPropertyNames; } + /** + * Adds the Flow Data's errors under the top-level key errors as a list + * of their messages, in the order the Flow Data holds them. A throwable + * with no message is written as its type, so the list holds only text. + * An error with no throwable has no message and is left out. + */ private void addErrors(FlowData data, Map allProperties) { - // If there are any errors then add them to the Json. - if (data.getErrors() != null && data.getErrors().size() > 0) { - Map> errors = new HashMap<>(); - for (FlowError error : data.getErrors()) - { - if (errors.containsKey(error.getFlowElement().getElementDataKey())) { - errors.get(error.getFlowElement().getElementDataKey()).add(error.getThrowable().getMessage()); - } else { - errors.put(error.getFlowElement().getElementDataKey(), - Collections.singletonList( - error.getThrowable().getMessage())); - } + Collection flowErrors = data.getErrors(); + if (flowErrors == null) { + return; + } + List messages = new ArrayList<>(); + for (FlowError error : flowErrors) { + Throwable throwable = error.getThrowable(); + if (throwable != null) { + String message = throwable.getMessage(); + messages.add(message != null ? + message : + throwable.getClass().getName()); } - allProperties.put("errors", errors); + } + if (messages.isEmpty() == false) { + allProperties.put("errors", messages); } } static String resolveName(String name) { - return name == JAVASCRIPT_PROPERTIES_NAME ? name : name.toLowerCase(); + return name == JAVASCRIPT_PROPERTIES_NAME ? name : name.toLowerCase(Locale.ROOT); } private String buildJson(Map allProperties) { @@ -460,7 +469,7 @@ private String buildJson(Map allProperties) { for (Map.Entry entry : allProperties.entrySet()) { if (entry.getValue() instanceof Map) { - Map map = new HashMap<>(); + JSONObject element = new JSONObject(); @SuppressWarnings("unchecked") Map properties = (Map)entry.getValue(); @@ -470,9 +479,9 @@ private String buildJson(Map allProperties) { if(value instanceof JavaScript) value = value.toString(); - map.put(resolveName(ent.getKey()), value); + element.put(resolveName(ent.getKey()), toJsonValue(value)); } - json.put(resolveName(entry.getKey()), map); + json.put(resolveName(entry.getKey()), element); } else { json.put(resolveName(entry.getKey()), entry.getValue()); @@ -481,6 +490,36 @@ private String buildJson(Map allProperties) { return json.toString(2); } + /** + * The value as it is written into the JSON. org.json leaves out a Map + * entry whose value is null, so a Map, such as an item of a list of + * Element Data, is written here with each null as JSON null, because a + * Property assigned no value is written as null with its nullreason. + * Every other value is wrapped as org.json wraps it. + */ + private static Object toJsonValue(Object value) { + if (value == null) { + return JSONObject.NULL; + } + if (value instanceof Map) { + JSONObject object = new JSONObject(); + for (Map.Entry entry : ((Map) value).entrySet()) { + object.put( + String.valueOf(entry.getKey()), + toJsonValue(entry.getValue())); + } + return object; + } + if (value instanceof Collection) { + JSONArray array = new JSONArray(); + for (Object item : (Collection) value) { + array.put(toJsonValue(item)); + } + return array; + } + return JSONObject.wrap(value); + } + /** * Executed on first request in order to build some collections @@ -495,7 +534,7 @@ private PipelineConfig populateMetaDataCollections(Pipeline pipeline) { pipeline.getElementAvailableProperties().entrySet()) { for (String propertyName : getDelayedPropertyNames( - element.getKey().toLowerCase(), + element.getKey().toLowerCase(Locale.ROOT), element.getValue().values())) { config.delayedExecutionProperties.add(propertyName); } @@ -509,8 +548,8 @@ private PipelineConfig populateMetaDataCollections(Pipeline pipeline) { pipeline.getElementAvailableProperties().entrySet()) { for (Map.Entry> property : getEvidencePropertyNames( config.delayedExecutionProperties, - element.getKey().toLowerCase(), - element.getKey().toLowerCase(), + element.getKey().toLowerCase(Locale.ROOT), + element.getKey().toLowerCase(Locale.ROOT), element.getValue().values()).entrySet()) { config.delayedEvidenceProperties.put( property.getKey(), @@ -535,7 +574,7 @@ private List getDelayedPropertyNames ( property.getType().equals(JavaScript.class)) { result.add(dataPath + EVIDENCE_SEPERATOR + - property.getName().toLowerCase()); + property.getName().toLowerCase(Locale.ROOT)); } // Call recursively for any properties that have sub-properties. @@ -567,7 +606,7 @@ private Map> getEvidencePropertyNames( for (String evidenceProperty : property.getEvidenceProperties()) { String evidenceName = elementDataKey + EVIDENCE_SEPERATOR + - evidenceProperty.toLowerCase(); + evidenceProperty.toLowerCase(Locale.ROOT); if (delayedExecutionProperties.contains(evidenceName)) { evidenceProperties.add(evidenceName); } @@ -577,7 +616,7 @@ private Map> getEvidencePropertyNames( // more evidence properties. if (evidenceProperties.size() > 0) { result.put(propertyDataPath + EVIDENCE_SEPERATOR + - property.getName().toLowerCase(), + property.getName().toLowerCase(Locale.ROOT), evidenceProperties); } diff --git a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderMultiPipelineTests.java b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderMultiPipelineTests.java new file mode 100644 index 000000000..0b16dc2a6 --- /dev/null +++ b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderMultiPipelineTests.java @@ -0,0 +1,150 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.jsonbuilder; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.ElementPropertyMetaDataDefault; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.data.PropertyMatcher; +import fiftyone.pipeline.core.data.TryGetResult; +import fiftyone.pipeline.core.data.types.JavaScript; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.jsonbuilder.data.JsonBuilderData; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElement; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.mockito.stubbing.Answer; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * One JSON Builder in two Pipelines keeps the configuration it reads from + * each Pipeline apart, and uses the right one for each. See + * multi-pipeline-elements.md line 17: + * "If this feature is enabled, then all Flow Elements that maintain any + * such state information MUST ensure that they have the means to keep + * track of isolated instances of this state for each Pipeline to which + * they are added and use the correct state information when processing." + */ +public class JsonBuilderMultiPipelineTests { + + /** + * A Pipeline whose "test" element has a JavaScript property, delayed or + * not. + */ + private static Pipeline pipeline(boolean delayExecution) { + FlowElement element = mock(FlowElement.class); + doReturn("test").when(element).getElementDataKey(); + ElementPropertyMetaData property = new ElementPropertyMetaDataDefault( + "jsproperty", + element, + "jsproperty", + JavaScript.class, + true, + null, + delayExecution, + null); + Map> metaData = + new HashMap<>(); + metaData.put("test", + Collections.singletonMap("jsproperty", property)); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElementAvailableProperties()).thenReturn(metaData); + return pipeline; + } + + /** + * Process Flow Data of the Pipeline with the element and return the + * JSON of the "test" element. + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + private static JSONObject process( + JsonBuilderElement element, + Pipeline pipeline) throws Exception { + ElementData data = mock(ElementData.class); + Map values = new HashMap<>(); + values.put("jsproperty", "var x = 1;"); + doReturn(values).when(data).asKeyMap(); + Map elementData = new HashMap<>(); + elementData.put("test", data); + + FlowData flowData = mock(FlowData.class); + when(flowData.elementDataAsMap()).thenReturn(elementData); + TryGetResult session = new TryGetResult<>(); + session.setValue("session"); + TryGetResult sequence = new TryGetResult<>(); + sequence.setValue(1); + when(flowData.tryGetEvidence(eq("query.session-id"), any(Class.class))) + .thenReturn(session); + when(flowData.tryGetEvidence(eq("query.sequence"), any(Class.class))) + .thenReturn(sequence); + when(flowData.getWhere(any(PropertyMatcher.class))) + .thenReturn(new HashMap<>()); + when(flowData.getPipeline()).thenReturn(pipeline); + when(flowData.getErrors()).thenReturn(new ArrayList()); + final JsonBuilderData[] result = { null }; + when(flowData.getOrAdd(anyString(), any(FlowElement.DataFactory.class))) + .thenAnswer((Answer) invocation -> { + FlowElement.DataFactory factory = invocation.getArgument(1); + result[0] = (JsonBuilderData) factory.create(flowData); + return result[0]; + }); + + element.process(flowData); + return new JSONObject(result[0].getJson()).getJSONObject("test"); + } + + @Test + public void onePipelinesConfiguration_IsNotUsedForAnother() + throws Exception { + JsonBuilderElement element = new JsonBuilderElementBuilder( + LoggerFactory.getILoggerFactory()).build(); + Pipeline delayed = pipeline(true); + Pipeline notDelayed = pipeline(false); + + assertTrue(process(element, delayed).has("jspropertydelayexecution"), + "The first Pipeline's property is delayed."); + assertFalse( + process(element, notDelayed).has("jspropertydelayexecution"), + "The second Pipeline's configuration came from the first."); + assertTrue(process(element, delayed).has("jspropertydelayexecution"), + "The first Pipeline's configuration came from the second."); + } +} diff --git a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java index fe9923ff7..15226b25b 100644 --- a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java +++ b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java @@ -73,7 +73,8 @@ public class JsonBuilderTests { private FlowData flowData; private JsonBuilderData result; private Pipeline pipeline; - + private List flowErrors; + public JsonBuilderTests() { ILoggerFactory internalLogger = mock(ILoggerFactory.class); when(internalLogger.getLogger(anyString())).thenReturn(mock(Logger.class)); @@ -114,6 +115,81 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { Map> propertyMetaData = new HashMap<>(); when(pipeline.getElementAvailableProperties()).thenReturn(propertyMetaData); + + flowErrors = new ArrayList<>(); + } + + /** + * The errors are a list of their messages under the top-level key + * errors, in the order the Flow Data holds them, whichever element + * recorded each. See the Process section of the JSON Builder + * specification. + */ + @Test + public void JsonBuilder_Errors_ListOfMessages() throws Exception { + FlowElement first = mock(FlowElement.class); + when(first.getElementDataKey()).thenReturn("first"); + FlowElement second = mock(FlowElement.class); + when(second.getElementDataKey()).thenReturn("second"); + flowErrors.add(new FlowError.Default( + new Exception("first element, one"), first)); + flowErrors.add(new FlowError.Default( + new Exception("second element"), second)); + flowErrors.add(new FlowError.Default( + new Exception("first element, two"), first)); + + String json = testIteration(1); + + JSONArray errors = new JSONObject(json).getJSONArray("errors"); + assertEquals( + Arrays.asList( + "first element, one", + "second element", + "first element, two"), + errors.toList(), + json); + } + + /** + * An error whose throwable has no message is written as the throwable's + * type, so the list holds only text. + */ + @Test + public void JsonBuilder_Errors_NoMessage_WritesType() throws Exception { + FlowElement element = mock(FlowElement.class); + when(element.getElementDataKey()).thenReturn("element"); + flowErrors.add(new FlowError.Default( + new IllegalStateException(), element)); + + String json = testIteration(1); + + JSONArray errors = new JSONObject(json).getJSONArray("errors"); + assertEquals( + Collections.singletonList(IllegalStateException.class.getName()), + errors.toList(), + json); + } + + /** + * An error with no throwable has no message to write, so it is left out, + * and with no messages at all the key errors is left out too. + */ + @Test + public void JsonBuilder_Errors_NoThrowable_LeftOut() throws Exception { + FlowElement element = mock(FlowElement.class); + flowErrors.add(new FlowError.Default(null, element)); + + String json = testIteration(1); + assertFalse(new JSONObject(json).has("errors"), json); + + flowErrors.add(new FlowError.Default( + new Exception("the message"), element)); + + json = testIteration(1); + assertEquals( + Collections.singletonList("the message"), + new JSONObject(json).getJSONArray("errors").toList(), + json); } /** @@ -127,6 +203,35 @@ public void JsonBuilder_ValidJson() throws Exception { assertTrue(isExpectedJson(json)); } + /** + * Element and property names are lower cased in the root locale, so a + * capital I keeps its dot under a Turkish default locale, which would + * otherwise lower it to a dotless i that the client script never reads. + */ + @Test + public void JsonBuilder_TurkishLocale_NamesLowerCasedInRootLocale() + throws Exception { + Map values = new HashMap<>(); + values.put("IsMobile", "True"); + ElementData element = mock(ElementData.class); + doReturn(values).when(element).asKeyMap(); + Map data = new HashMap<>(); + data.put("IpData", element); + + Locale original = Locale.getDefault(); + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + String json; + try { + json = testIteration(1, data); + } finally { + Locale.setDefault(original); + } + + JSONObject obj = new JSONObject(json); + assertTrue(obj.has("ipdata"), json); + assertEquals("True", obj.getJSONObject("ipdata").opt("ismobile"), json); + } + /** * Check that the JSON element removes JavaScript properties from the * response after max number of iterations has been reached. @@ -551,6 +656,67 @@ public void JsonBuilder_WeightedValues() throws Exception { } } + /** + * A Property assigned no value is written as null with its nullreason, + * and a Property whose value is null without being assigned no value has + * no entry and no nullreason. See the Process section of the JSON + * Builder specification. + */ + @Test + public void JsonBuilder_NoValue_WrittenAsNullWithReason() + throws Exception { + AspectPropertyValueDefault noValue = + new AspectPropertyValueDefault<>(); + noValue.setNoValueMessage("There is no value."); + Map map = new HashMap<>(); + map.put("novalue", noValue); + map.put("plainnull", null); + map.put("property", "thisIsAValue"); + when(elementDataMock.asKeyMap()).thenReturn(map); + + String json = testIteration(1); + + JSONObject element = new JSONObject(json).getJSONObject("test"); + assertTrue(element.has("novalue"), json); + assertTrue(element.isNull("novalue"), json); + assertEquals("There is no value.", + element.getString("novaluenullreason"), json); + assertFalse(element.has("plainnull"), json); + assertFalse(element.has("plainnullnullreason"), json); + assertEquals("thisIsAValue", element.getString("property"), json); + } + + /** + * Each item of a list of Element Data follows the same rules, so an + * item's Property assigned no value is written as null with its + * nullreason. + */ + @Test + public void JsonBuilder_ListOfElementData_NoValueWrittenAsNull() + throws Exception { + AspectPropertyValueDefault noValue = + new AspectPropertyValueDefault<>(); + noValue.setNoValueMessage("No value for this item."); + Map itemValues = new HashMap<>(); + itemValues.put("value1", noValue); + itemValues.put("value2", "xyz"); + ElementData item = mock(ElementData.class); + when(item.asKeyMap()).thenReturn(itemValues); + Map map = new HashMap<>(); + map.put("items", Collections.singletonList(item)); + when(elementDataMock.asKeyMap()).thenReturn(map); + + String json = testIteration(1); + + JSONObject first = new JSONObject(json).getJSONObject("test") + .getJSONArray("items").getJSONObject(0); + assertTrue(first.has("value1"), json); + assertTrue(first.isNull("value1"), json); + assertEquals("No value for this item.", + first.getString("value1nullreason"), json); + assertEquals("xyz", first.getString("value2"), json); + } + private String testIteration(int iteration) throws Exception { return testIteration(iteration, null); } @@ -598,6 +764,7 @@ private String testIteration( return result[0]; }); when(flowData.getPipeline()).thenReturn(pipeline); + when(flowData.getErrors()).thenReturn(flowErrors); jsonBuilderElement.process(flowData); diff --git a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/WritesAfterProcessingTests.java b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/WritesAfterProcessingTests.java new file mode 100644 index 000000000..929958300 --- /dev/null +++ b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/WritesAfterProcessingTests.java @@ -0,0 +1,83 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.jsonbuilder; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElement; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * The check, for the Flow Element in this module, the + * {@link JsonBuilderElement}, that processes it, then writes to its Element + * Data and expects the error. + * @see Specification + */ +public class WritesAfterProcessingTests { + + private static final ILoggerFactory LOGGER_FACTORY = + LoggerFactory.getILoggerFactory(); + + static void assertWritesFail(ElementData data) { + assertThrows( + IllegalStateException.class, + () -> data.put("after", "value"), + "put"); + assertThrows( + IllegalStateException.class, + () -> data.populateFromMap( + Collections.singletonMap("after", "value")), + "populateFromMap"); + } + + @Test + public void jsonBuilder_ElementDataRefusesWrites() throws Exception { + JsonBuilderElement jsonBuilder = + new JsonBuilderElementBuilder(LOGGER_FACTORY).build(); + try (Pipeline pipeline = new PipelineBuilder(LOGGER_FACTORY) + .addFlowElement( + new SequenceElementBuilder(LOGGER_FACTORY).build()) + .addFlowElement(jsonBuilder) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + data.process(); + assertEquals(0, data.getErrors().size(), + String.valueOf(data.getErrors())); + + ElementData own = data.getFromElement(jsonBuilder); + assertWritesFail(own); + } + } +} diff --git a/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBase.java b/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBase.java index 624c7b116..5ec346535 100644 --- a/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBase.java +++ b/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBase.java @@ -46,6 +46,7 @@ import java.util.Arrays; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; @@ -68,6 +69,10 @@ * Only string based types are supported for translation (string, list of * strings, weighted list of strings and the {@code AspectPropertyValue} * wrappers); the output type matches the input. + *

    + * Make a translation engine through its builder, as every Flow Element is + * made. The class stays concrete with a public constructor because released + * code may make one directly. * * @param the type of element data the engine writes */ @@ -206,7 +211,7 @@ public List getProperties() { Set seen = new LinkedHashSet<>(); for (TranslationProperty translation : translationProperties) { String name = translation.getDestinationProperty(); - if (seen.add(name.toLowerCase())) { + if (seen.add(name.toLowerCase(Locale.ROOT))) { result.add(new ElementPropertyMetaDataDefault( name, this, "", Object.class, true)); } diff --git a/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBuilder.java b/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBuilder.java index 3c460191d..39e157c75 100644 --- a/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBuilder.java +++ b/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBuilder.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.translation.flowelements; +import fiftyone.pipeline.annotations.DefaultValue; import fiftyone.pipeline.annotations.ElementBuilder; import fiftyone.pipeline.translation.data.MissingTranslationBehavior; import fiftyone.pipeline.translation.data.TranslationData; @@ -80,6 +81,7 @@ public TranslationEngineBuilder(ILoggerFactory loggerFactory) { * @param sourceElementDataKey the source element data key * @return this builder */ + @DefaultValue("No default - the source element data key must be supplied") public TranslationEngineBuilder setSourceElementDataKey( String sourceElementDataKey) { this.sourceElementDataKey = sourceElementDataKey; @@ -93,6 +95,7 @@ public TranslationEngineBuilder setSourceElementDataKey( * @param language the language to translate to * @return this builder */ + @DefaultValue("None - the language is resolved from the evidence") public TranslationEngineBuilder setFixedLanguage(String language) { this.fixedLanguage = language; return this; @@ -103,6 +106,7 @@ public TranslationEngineBuilder setFixedLanguage(String language) { * @param behavior the missing-translation behaviour * @return this builder */ + @DefaultValue("ORIGINAL") public TranslationEngineBuilder setMissingTranslationBehavior( MissingTranslationBehavior behavior) { this.behavior = behavior; diff --git a/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/BuilderDefaultValueTests.java b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/BuilderDefaultValueTests.java new file mode 100644 index 000000000..4d28b2448 --- /dev/null +++ b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/BuilderDefaultValueTests.java @@ -0,0 +1,68 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.translation.flowelements; + +import fiftyone.pipeline.annotations.DefaultValue; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Every configuration setter of the translation engine builder gives its + * default value beside it, with {@link DefaultValue}. See + * reference-implementation-notes.md line 79: + * "In all cases, there SHOULD be some mechanism to specify the default + * values alongside configuration property setters." + */ +public class BuilderDefaultValueTests { + + static List missingDefaults(Class... builders) { + List missing = new ArrayList<>(); + for (Class builder : builders) { + for (Method method : builder.getDeclaredMethods()) { + if (Modifier.isPublic(method.getModifiers()) && + method.getName().startsWith("set") && + method.isSynthetic() == false && + method.isBridge() == false && + method.getAnnotation(DefaultValue.class) == null) { + missing.add(builder.getSimpleName() + "#" + + method.getName() + + Arrays.toString(method.getParameterTypes())); + } + } + } + return missing; + } + + @Test + public void translationEngineBuilderSetters_GiveTheirDefaults() { + assertEquals(new ArrayList(), + missingDefaults(TranslationEngineBuilder.class)); + } +} diff --git a/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBasePublicTests.java b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBasePublicTests.java new file mode 100644 index 000000000..391e756c7 --- /dev/null +++ b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBasePublicTests.java @@ -0,0 +1,57 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.translation.flowelements; + +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Modifier; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * TranslationEngineBase is a released public class with a public + * constructor, so a patch release keeps it concrete and keeps that + * constructor, and code that makes one with {@code new} still compiles. + */ +public class TranslationEngineBasePublicTests { + + @Test + public void translationEngineBase_IsConcrete() { + assertFalse( + Modifier.isAbstract(TranslationEngineBase.class.getModifiers()), + "TranslationEngineBase is abstract, so code that makes one " + + "with new no longer compiles"); + } + + @Test + public void translationEngineBase_HasPublicConstructor() { + boolean found = false; + for (Constructor constructor : + TranslationEngineBase.class.getConstructors()) { + found |= Modifier.isPublic(constructor.getModifiers()); + } + assertTrue(found, "TranslationEngineBase has no public constructor"); + } +} diff --git a/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/WritesAfterProcessingTests.java b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/WritesAfterProcessingTests.java new file mode 100644 index 000000000..61e21b415 --- /dev/null +++ b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/WritesAfterProcessingTests.java @@ -0,0 +1,89 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.translation.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * The check, for the Flow Elements in this module, that processes each one, + * then writes to its Element Data and expects the error. The + * {@link TranslationEngine} is the {@link TranslationEngineBase} that the + * module's builder makes. + * @see Specification + */ +public class WritesAfterProcessingTests { + + private static final ILoggerFactory LOGGER_FACTORY = + LoggerFactory.getILoggerFactory(); + + static void assertWritesFail(ElementData data) { + assertThrows( + IllegalStateException.class, + () -> data.put("after", "value"), + "put"); + assertThrows( + IllegalStateException.class, + () -> data.populateFromMap( + Collections.singletonMap("after", "value")), + "populateFromMap"); + } + + @Test + public void translationEngine_ElementDataRefusesWrites() + throws Exception { + TranslationEngine translation = + new TranslationEngineBuilder(LOGGER_FACTORY) + .addSource("countries.fr_FR.yml", "England: Angleterre") + .addTranslation("Country", "CountryTranslated") + .setSourceElementDataKey("source") + .setFixedLanguage("fr_FR") + .build(); + try (Pipeline pipeline = new PipelineBuilder(LOGGER_FACTORY) + .addFlowElement(new CountrySourceElementBuilder() + .setCountry("England") + .build()) + .addFlowElement(translation) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + data.process(); + assertEquals(0, data.getErrors().size(), + String.valueOf(data.getErrors())); + + ElementData own = data.getFromElement(translation); + assertEquals("Angleterre", own.get("CountryTranslated")); + assertWritesFail(own); + } + } +} diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java index 61022376d..1e19a9427 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java @@ -58,6 +58,11 @@ * This carries out the processing on the device making the request using the * Pipeline which has been constructed, and intercepts requests for the * 51Degrees JavaScript. + *

    + * A Flow Data cannot be used by two threads at once. Application code reads + * the values it needs from the Flow Data on the request's thread, and hands + * those values, not the Flow Data, to any other thread or task. The Flow + * Data is closed when the request ends. */ @Component public class FiftyOneInterceptor extends HandlerInterceptorAdapter { @@ -122,6 +127,8 @@ public FiftyOneInterceptor( resultService.setPipeline(pipeline); clientsidePropertyService.setPipeline(pipeline); + clientsidePropertyService.setCacheMaxAge( + config.getClientsideCacheMaxAge()); fiftyOneJsService.enable(config.getClientsidePropertiesEnabled()); fiftyOneJsService.setPaths(config.getJavaScriptPath(), jsonPath); } @@ -162,6 +169,14 @@ public boolean preHandle( HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { + // A request for the 51Degrees JavaScript or JSON whose If-None-Match + // matches the ETag of its Evidence is answered without processing, + // because that Evidence alone decides the content. Nothing is left + // to close. + if (fiftyOneJsService.serveNotModified( + request, response, resultService)) { + return false; + } resultService.process(request); boolean proceed = false; diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FlowDataProvider.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FlowDataProvider.java index 0c2585da2..1c0d23d61 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FlowDataProvider.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FlowDataProvider.java @@ -27,6 +27,11 @@ /** * Spring framework component for the {@link FlowDataProviderCore}. + *

    + * A Flow Data cannot be used by two threads at once. Application code reads + * the values it needs from the Flow Data on the request's thread, and hands + * those values, not the Flow Data, to any other thread or task. The Flow + * Data is closed when the request ends. */ @Component public interface FlowDataProvider extends FlowDataProviderCore { diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfig.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfig.java index 5dcdf3cd2..712e487b5 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfig.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfig.java @@ -24,6 +24,8 @@ import fiftyone.pipeline.web.mvc.components.FiftyOneInterceptor; +import static fiftyone.pipeline.web.Constants.DEFAULT_CLIENTSIDE_CACHE_MAX_AGE_SECONDS; + /** * Configuration class for the {@link FiftyOneInterceptor}. */ @@ -41,6 +43,15 @@ public interface FiftyOneInterceptorConfig { */ boolean getClientsidePropertiesEnabled(); + /** + * Get how many seconds a browser may keep the client-side JavaScript or + * JSON before it asks again, the max-age of their Cache-Control header. + * @return the max-age in seconds, 0 or more + */ + default int getClientsideCacheMaxAge() { + return DEFAULT_CLIENTSIDE_CACHE_MAX_AGE_SECONDS; + } + /** * Get the end of the request path the JavaScript is served for. * @return the path, or null or empty for the default diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfigDefault.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfigDefault.java index bd22601cf..3bd000b66 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfigDefault.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfigDefault.java @@ -23,6 +23,8 @@ package fiftyone.pipeline.web.mvc.configuration; import javax.servlet.ServletContext; + +import static fiftyone.pipeline.web.Constants.DEFAULT_CLIENTSIDE_CACHE_MAX_AGE_SECONDS; import org.springframework.beans.factory.annotation.Autowired; /** @@ -34,6 +36,9 @@ public class FiftyOneInterceptorConfigDefault implements FiftyOneInterceptorConf private boolean clientsidePropertiesEnabled; + private int clientsideCacheMaxAge = + DEFAULT_CLIENTSIDE_CACHE_MAX_AGE_SECONDS; + private String javaScriptPath; private String jsonPath; @@ -53,6 +58,21 @@ public void setClientsidePropertiesEnabled(boolean clientsidePropertiesEnabled) this.clientsidePropertiesEnabled = clientsidePropertiesEnabled; } + /** + * Set how many seconds a browser may keep the client-side JavaScript or + * JSON before it asks again, the max-age of their Cache-Control header. + * @param seconds the max-age, 0 or more + * @throws IllegalArgumentException if seconds is negative + */ + public void setClientsideCacheMaxAge(int seconds) { + if (seconds < 0) { + throw new IllegalArgumentException( + "The max-age of the client-side responses cannot be " + + "negative, and was " + seconds + "."); + } + this.clientsideCacheMaxAge = seconds; + } + public void setJavaScriptPath(String javaScriptPath) { this.javaScriptPath = javaScriptPath; } @@ -98,4 +118,9 @@ public String getDataFilePath() { public boolean getClientsidePropertiesEnabled() { return clientsidePropertiesEnabled; } + + @Override + public int getClientsideCacheMaxAge() { + return clientsideCacheMaxAge; + } } diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/services/ClientsidePropertyService.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/services/ClientsidePropertyService.java index cc026b3b3..83de0f8f5 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/services/ClientsidePropertyService.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/services/ClientsidePropertyService.java @@ -38,6 +38,15 @@ @Service public interface ClientsidePropertyService extends ClientsidePropertyServiceCore { void setPipeline(Pipeline pipeline); + + /** + * Set how many seconds a browser may keep a good response before it + * asks again, the max-age of the Cache-Control header. An + * implementation that sets no Cache-Control header ignores it. + * @param seconds the max-age, 0 or more + */ + default void setCacheMaxAge(int seconds) { + } void serveJavascript( HttpServletRequest request, HttpServletResponse response) throws IOException; diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java index 3f000fa3b..364db9d32 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java @@ -30,8 +30,14 @@ import static org.mockito.Mockito.*; import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -53,6 +59,10 @@ import fiftyone.pipeline.web.mvc.services.FiftyOneJSService; import fiftyone.pipeline.web.mvc.services.PipelineResultService; import fiftyone.pipeline.web.mvc.services.UACHService; +import fiftyone.pipeline.web.mvc.services.WebRequestEvidenceService; + +import static fiftyone.pipeline.web.Constants.CORE_JS_NAME; +import static fiftyone.pipeline.web.Constants.HTTPCONTEXT_FLOWDATA; public class FiftyOneInterceptorTests { private HttpServletRequest request; @@ -107,6 +117,31 @@ public void init() throws IOException { uachService); } + /** + * The interceptor gives the client-side service the max-age its + * configuration sets. + */ + @Test + public void FiftyOneInterceptor_SetsTheCacheMaxAge() { + FiftyOneInterceptorConfig config = + mock(FiftyOneInterceptorConfig.class); + when(config.getDataFilePath()) + .thenReturn("src/test/resources/Test.xml"); + when(config.getClientsideCacheMaxAge()).thenReturn(60); + ClientsidePropertyService clientsidePropertyService = + mock(ClientsidePropertyService.class); + + new FiftyOneInterceptor( + config, + mock(PipelineResultService.class), + mock(FlowDataProvider.class), + clientsidePropertyService, + mock(FiftyOneJSService.class), + mock(UACHService.class)); + + verify(clientsidePropertyService, times(1)).setCacheMaxAge(60); + } + /** * Check that 'resultService.process()' is called during * interceptor's preHandle process @@ -149,6 +184,68 @@ public void FiftyOneInterceptor_PreHandle_ServesJavaScript_ClosesFlowData() .close(); } + /** + * A request for the JavaScript whose If-None-Match matches the ETag of + * the Evidence it supplies is answered with a 304 without processing, as + * the servlet filter does. + */ + @Test + public void FiftyOneInterceptor_MatchingIfNoneMatch_304_NotProcessed() + throws Exception { + FiftyOneInterceptorConfig config = + mock(FiftyOneInterceptorConfig.class); + when(config.getDataFilePath()) + .thenReturn("src/test/resources/Test.xml"); + when(config.getClientsidePropertiesEnabled()).thenReturn(true); + when(config.getClientsideCacheMaxAge()).thenReturn(1800); + when(config.getSequenceElementEnabled()).thenReturn(true); + when(config.getSetHeadersElementEnabled()).thenReturn(true); + FlowDataProvider provider = new FlowDataProvider.Default(); + ClientsidePropertyService clientside = + new ClientsidePropertyService.Default(provider); + FiftyOneInterceptor real = new FiftyOneInterceptor( + config, + new PipelineResultService.Default( + new WebRequestEvidenceService.Default()), + provider, + clientside, + new FiftyOneJSService.Default(clientside), + new UACHService.Default(provider)); + + Map attributes = new HashMap<>(); + HttpServletRequest revalidation = mock(HttpServletRequest.class); + when(revalidation.getMethod()).thenReturn("GET"); + when(revalidation.getRequestURI()).thenReturn("/" + CORE_JS_NAME); + when(revalidation.getContextPath()).thenReturn(""); + when(revalidation.getScheme()).thenReturn("https"); + when(revalidation.getHeaderNames()).thenAnswer(invocation -> + Collections.enumeration(Arrays.asList("Host", "If-None-Match"))); + when(revalidation.getHeader("Host")).thenReturn("example.com"); + when(revalidation.getHeader("If-None-Match")).thenReturn("*"); + when(revalidation.getHeaders("If-None-Match")).thenAnswer(invocation -> + Collections.enumeration(Collections.singletonList("*"))); + doAnswer(invocation -> attributes.put( + invocation.getArgument(0), invocation.getArgument(1))) + .when(revalidation).setAttribute(anyString(), any()); + when(revalidation.getAttribute(anyString())).thenAnswer(invocation -> + attributes.get((String) invocation.getArgument(0))); + HttpServletResponse answer = mock(HttpServletResponse.class); + when(answer.getWriter()) + .thenReturn(new PrintWriter(new StringWriter())); + + try { + assertFalse(real.preHandle(revalidation, answer, handler)); + } finally { + real.pipeline.close(); + } + + verify(revalidation, never()).setAttribute( + eq(HTTPCONTEXT_FLOWDATA), any()); + verify(answer).setStatus(304); + verify(answer).setHeader( + eq("Cache-Control"), eq("private,max-age=1800")); + } + private FiftyOneInterceptor newInterceptor( FiftyOneInterceptorConfig config, FiftyOneJSService jsService) throws IOException { diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfigDefaultTests.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfigDefaultTests.java new file mode 100644 index 000000000..6e834cbb3 --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfigDefaultTests.java @@ -0,0 +1,60 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web.mvc.configuration; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class FiftyOneInterceptorConfigDefaultTests { + + /** + * A negative max-age is refused where it is set, rather than when the + * interceptor starts. + */ + @Test + public void setClientsideCacheMaxAge_Negative_Throws() { + FiftyOneInterceptorConfigDefault config = + new FiftyOneInterceptorConfigDefault(); + try { + config.setClientsideCacheMaxAge(-1); + fail("An IllegalArgumentException should have been thrown"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().contains("-1")); + } + } + + /** + * A max-age of 0 is kept, as it tells the browser to ask again on each + * use. + */ + @Test + public void setClientsideCacheMaxAge_Zero_Kept() { + FiftyOneInterceptorConfigDefault config = + new FiftyOneInterceptorConfigDefault(); + config.setClientsideCacheMaxAge(0); + assertEquals(0, config.getClientsideCacheMaxAge()); + } +} diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/Constants.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/Constants.java index 7298dd10e..70efca179 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/Constants.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/Constants.java @@ -81,6 +81,20 @@ public class Constants { */ public static final boolean DEFAULT_CLIENTSIDE_ENABLED = true; + /** + * Default number of seconds a browser may keep the client-side + * JavaScript or JSON before it asks again, the max-age of their + * Cache-Control header. + */ + public static final int DEFAULT_CLIENTSIDE_CACHE_MAX_AGE_SECONDS = 1800; + + /** + * Name of the filter init parameter that sets the max-age of the + * client-side endpoints' Cache-Control header, in seconds. + */ + public static final String INIT_PARAM_CLIENTSIDE_CACHE_MAX_AGE = + "clientside-cache-max-age"; + /** * Default value for the options that add the Sequence Element and the * Set Headers Element. diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java index b1090a6ef..f1ef653d4 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java @@ -42,9 +42,11 @@ import static fiftyone.pipeline.web.Constants.DEFAULT_JSON_PATH; import static fiftyone.pipeline.web.Constants.CORE_JS_NAME; +import static fiftyone.pipeline.web.Constants.DEFAULT_CLIENTSIDE_CACHE_MAX_AGE_SECONDS; import static fiftyone.pipeline.web.Constants.DEFAULT_CLIENTSIDE_ENABLED; import static fiftyone.pipeline.web.Constants.DEFAULT_CONFIG_FILE; import static fiftyone.pipeline.web.Constants.DEFAULT_ELEMENT_ENABLED; +import static fiftyone.pipeline.web.Constants.INIT_PARAM_CLIENTSIDE_CACHE_MAX_AGE; import static fiftyone.pipeline.web.Constants.INIT_PARAM_JAVASCRIPT_PATH; import static fiftyone.pipeline.web.Constants.INIT_PARAM_JSON_PATH; import static fiftyone.pipeline.web.Constants.INIT_PARAM_SEQUENCE_ELEMENT_ENABLED; @@ -53,9 +55,17 @@ /** * Servlet filter used to intercept HTTP requests and process them using the * 51Degrees Pipeline. + *

    + * A Flow Data cannot be used by two threads at once. Application code reads + * the values it needs from the Flow Data on the request's thread, and hands + * those values, not the Flow Data, to any other thread or task. The Flow + * Data is closed when the request ends. * @see Specification */ public class PipelineFilter implements Filter { + private static final org.slf4j.Logger logger = + LoggerFactory.getLogger(PipelineFilter.class); + private Pipeline pipeline; private WebRequestEvidenceServiceCore evidenceService; @@ -87,6 +97,7 @@ public void init(FilterConfig filterConfig) throws ServletException { else { clientsideEnabled = Boolean.parseBoolean(clientsideEnabledString); } + int cacheMaxAgeSeconds = getCacheMaxAgeSeconds(filterConfig); boolean sequenceElementEnabled = getBooleanInitParameter( filterConfig, @@ -126,9 +137,12 @@ public void init(FilterConfig filterConfig) throws ServletException { resultService = new PipelineResultServiceCore.Default(evidenceService, pipeline); flowDataProviderCore = new FlowDataProviderCore.Default(); uachServiceCore = new UACHServiceCore.Default(); - clientsidePropertyServiceCore = new ClientsidePropertyServiceCore.Default( - flowDataProviderCore, - pipeline); + ClientsidePropertyServiceCore.Default clientside = + new ClientsidePropertyServiceCore.Default( + flowDataProviderCore, + pipeline); + clientside.setCacheMaxAge(cacheMaxAgeSeconds); + clientsidePropertyServiceCore = clientside; jsService = new FiftyOneJSServiceCore.Default( clientsidePropertyServiceCore, clientsideEnabled, @@ -153,38 +167,161 @@ private static String getPathInitParameter( defaultValue : value.trim(); } + /** + * The max-age of the client-side endpoints' Cache-Control header, in + * seconds, from the filter's init parameter, or the default when the + * parameter is not set. + * @param filterConfig the filter's configuration + * @return the max-age in seconds + * @throws ServletException if the parameter is not a whole number of + * seconds, 0 or more + */ + static int getCacheMaxAgeSeconds(FilterConfig filterConfig) + throws ServletException { + String value = filterConfig.getInitParameter( + INIT_PARAM_CLIENTSIDE_CACHE_MAX_AGE); + if (value == null || value.trim().isEmpty()) { + return DEFAULT_CLIENTSIDE_CACHE_MAX_AGE_SECONDS; + } + try { + int seconds = Integer.parseInt(value.trim()); + if (seconds >= 0) { + return seconds; + } + } catch (NumberFormatException e) { + // Reported below, as a negative number is. + } + throw new ServletException("The '" + + INIT_PARAM_CLIENTSIDE_CACHE_MAX_AGE + "' init parameter must " + + "be a whole number of seconds, 0 or more, and was '" + value + + "'."); + } + @Override public void doFilter( ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { - // Populate the request properties and store against the - // HttpContext. - resultService.process((HttpServletRequest)request); + HttpServletRequest httpRequest = (HttpServletRequest) request; + HttpServletResponse httpResponse = (HttpServletResponse) response; + boolean suppress = isSuppressProcessExceptions(); + boolean served; try { - // Get FlowData from the request. - FlowData flowData = flowDataProviderCore.getFlowData((HttpServletRequest)request); - - // Set UACH response headers. - uachServiceCore.setResponseHeaders( - flowData, (HttpServletResponse)response); - - // If 51Degrees JavaScript or JSON is being requested then serve it. - // Otherwise continue down the filter Pipeline. - if (jsService.serveJS((HttpServletRequest)request, (HttpServletResponse) response) == false && - jsService.serveJson((HttpServletRequest)request, (HttpServletResponse) response) == false) { - chain.doFilter(request, response);//sends request to next resource + served = processRequest(httpRequest, httpResponse); + } catch (RuntimeException e) { + // The Pipeline's setting decides whether an error in the web + // integration stops the request. When exceptions are suppressed + // it is logged and the request goes on without the results, so + // the API does not take down the website. See + // https://github.com/51Degrees/specifications/blob/706de1f0cb3edd560eb7fd7ef3bb19b93a4883dc/pipeline-specification/features/exception-handling.md#L73 + closeFlowData(httpRequest, suppress, e); + if (suppress == false) { + throw e; } + logger.error("The 51Degrees Pipeline could not process the " + + "request, which goes on without its results.", e); + served = false; + } catch (IOException e) { + // Writing the JavaScript or JSON failed, for example because the + // client went away. The request ends here, so the FlowData is + // closed before the failure is thrown. + closeFlowData(httpRequest, suppress, e); + throw e; + } + if (served) { + closeFlowData(httpRequest, suppress, null); + return; + } + + try { + chain.doFilter(request, response);//sends request to next resource } finally { // The rest of the filters in the chain have now been called, or // one of them failed. Either way it is time to dispose of the // FlowData instance. - try { - flowDataProviderCore.getFlowData((HttpServletRequest) request).close(); - } catch (Exception e) { - throw new ServletException("FlowData could not be disposed of.", e); + closeFlowData(httpRequest, suppress, null); + } + } + + /** + * Whether the Pipeline suppresses process exceptions, which also + * decides whether an error in this filter stops the request. + * @return true if exceptions are suppressed and logged + */ + private boolean isSuppressProcessExceptions() { + return pipeline != null && pipeline.isSuppressProcessExceptions(); + } + + /** + * Process the request and set the response headers, and serve the + * 51Degrees JavaScript or JSON if that is what is asked for. + * @param request the request + * @param response the response + * @return true if the response has been served, so the request does not + * go on down the filter chain + * @throws IOException if the JavaScript or JSON could not be written + */ + private boolean processRequest( + HttpServletRequest request, + HttpServletResponse response) throws IOException { + // A request for the 51Degrees JavaScript or JSON whose If-None-Match + // matches the ETag of its Evidence is answered without processing, + // because that Evidence alone decides the content. + if (jsService.serveNotModified(request, response, resultService)) { + return true; + } + + // Populate the request properties and store against the + // HttpContext. + resultService.process(request); + + // Get FlowData from the request. + FlowData flowData = flowDataProviderCore.getFlowData(request); + + // Set UACH response headers. + uachServiceCore.setResponseHeaders(flowData, response); + + // If 51Degrees JavaScript or JSON is being requested then serve it. + // Otherwise the request continues down the filter chain. + return jsService.serveJS(request, response) || + jsService.serveJson(request, response); + } + + /** + * Close the request's FlowData, if processing put one there. + * @param request the request + * @param suppress true if a failure to close is logged rather than + * thrown + * @param failure the failure already being handled, which a failure to + * close is added to, or null + * @throws ServletException if the FlowData could not be closed and + * exceptions are not suppressed + */ + private void closeFlowData( + HttpServletRequest request, + boolean suppress, + Exception failure) throws ServletException { + FlowData flowData; + try { + flowData = flowDataProviderCore.getFlowData(request); + } catch (RuntimeException e) { + flowData = null; + } + if (flowData == null) { + return; + } + try { + flowData.close(); + } catch (Exception e) { + if (failure != null) { + failure.addSuppressed(e); + } else if (suppress) { + logger.error("FlowData could not be disposed of.", e); + } else { + throw new ServletException( + "FlowData could not be disposed of.", e); } } } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/StartupHelpers.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/StartupHelpers.java index 7e13e8a2a..fb6582b59 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/StartupHelpers.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/StartupHelpers.java @@ -32,6 +32,8 @@ import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElement; +import java.util.Locale; + /** * Static methods used on server startup to configure and build the Pipeline and * services needed. @@ -50,7 +52,7 @@ private static int getElementIndex( String name) { int index = 0; for (ElementOptions element : options.elements) { - if (element.builderName.toLowerCase().contains(name.toLowerCase())) { + if (element.builderName.toLowerCase(Locale.ROOT).contains(name.toLowerCase(Locale.ROOT))) { return index; } index++; @@ -58,6 +60,23 @@ private static int getElementIndex( return -1; } + /** + * Add an element at the end of the configuration, but before a usage + * sharing element, which must stay the last Flow Element. + * @param options the options to add to + * @param element the element to add + */ + private static void addBeforeUsageSharing( + PipelineOptions options, + ElementOptions element) { + int index = getElementIndex(options, "ShareUsage"); + if (index >= 0 && index == options.elements.size() - 1) { + options.elements.add(index, element); + } else { + options.elements.add(element); + } + } + /** * Configure the extra web elements required and build the Pipeline using * the {@link PipelineBuilder#buildFromConfiguration(PipelineOptions)} @@ -184,7 +203,7 @@ public static Pipeline buildFromConfiguration( jsIndex++; } else { - options.elements.add(jsonBuilderElement); + addBeforeUsageSharing(options, jsonBuilderElement); } } @@ -196,7 +215,7 @@ public static Pipeline buildFromConfiguration( if(contextRoot != null) { javaScriptBuilderElement.buildParameters.put("ContextRoot", contextRoot); } - options.elements.add(javaScriptBuilderElement); + addBeforeUsageSharing(options, javaScriptBuilderElement); } else { // There is already a JavaScript builder config so check if @@ -238,7 +257,7 @@ public static Pipeline buildFromConfiguration( // The set-headers element is not included so add it. ElementOptions setHeadersElement = new ElementOptions(); setHeadersElement.builderName = SetHeadersElement.class.getSimpleName(); - options.elements.add(setHeadersElement); + addBeforeUsageSharing(options, setHeadersElement); } return builder.buildFromConfiguration(options); diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/ClientsidePropertyServiceCore.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/ClientsidePropertyServiceCore.java index d51d2cfe9..596c1c690 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/ClientsidePropertyServiceCore.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/ClientsidePropertyServiceCore.java @@ -25,20 +25,31 @@ import fiftyone.pipeline.core.data.EvidenceKeyFilter; import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.fiftyone.data.ShareUsageEvidence; import fiftyone.pipeline.javascriptbuilder.flowelements.*; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collection; +import java.util.Enumeration; import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TreeMap; import static fiftyone.pipeline.core.Constants.EVIDENCE_HTTPHEADER_PREFIX; import static fiftyone.pipeline.core.Constants.EVIDENCE_SEPERATOR; +import static fiftyone.pipeline.web.Constants.DEFAULT_CLIENTSIDE_CACHE_MAX_AGE_SECONDS; import fiftyone.pipeline.javascriptbuilder.data.JavaScriptBuilderData; import fiftyone.pipeline.jsonbuilder.data.JsonBuilderData; import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElement; @@ -82,6 +93,39 @@ void serveJson( HttpServletRequest request, HttpServletResponse response) throws IOException; + /** + * Answer a request for the JavaScript whose If-None-Match matches the + * ETag of the Evidence in the {@link FlowData}, which has not been + * processed. A GET or HEAD gets a 304 carrying the headers of a good + * response, and any other method a 412. + * @param flowData the request's {@link FlowData}, holding its Evidence + * @param request the {@link HttpServletRequest} to answer + * @param response the {@link HttpServletResponse} to write the answer to + * @return true if the request was answered, false if it is to be + * processed + */ + default boolean serveJavascriptNotModified( + FlowData flowData, + HttpServletRequest request, + HttpServletResponse response) { + return false; + } + + /** + * As {@link #serveJavascriptNotModified}, for the JSON. + * @param flowData the request's {@link FlowData}, holding its Evidence + * @param request the {@link HttpServletRequest} to answer + * @param response the {@link HttpServletResponse} to write the answer to + * @return true if the request was answered, false if it is to be + * processed + */ + default boolean serveJsonNotModified( + FlowData flowData, + HttpServletRequest request, + HttpServletResponse response) { + return false; + } + /** * Default implementation of the {@link ClientsidePropertyServiceCore} * interface. @@ -126,11 +170,25 @@ private enum ContentTypes { private List headersAffectingJavaScript; /** - * The cache control values that will be set for the JavaScript. + * The cache control value for a good response, which only the + * browser may keep, for the configured max-age. + */ + private volatile String cacheControl = + cacheControl(DEFAULT_CLIENTSIDE_CACHE_MAX_AGE_SECONDS); + + /** + * The cache control value for a response built while processing + * recorded errors. A browser may keep it but must ask before using + * it again, and with no ETag the next request gets a full response. */ - private final List cacheControl = Arrays.asList( - "private", - "max-age=1800"); + private static final String FAILED_CACHE_CONTROL = "private, no-cache"; + + /** + * The digest the ETag holds. Every Java platform provides it. + */ + private static final String ETAG_DIGEST = "SHA-256"; + + private static final char[] HEX = "0123456789abcdef".toCharArray(); /** * Create a new instance. @@ -201,6 +259,28 @@ private List getHeadersAffectingJavaScript() { return headersAffectingJavaScript; } + /** + * Set how many seconds a browser may keep a good response before it + * asks again, the max-age of the Cache-Control header. The default + * is {@link fiftyone.pipeline.web.Constants#DEFAULT_CLIENTSIDE_CACHE_MAX_AGE_SECONDS}. + * A shorter time suits JavaScript that gathers values which change, + * such as a location. + * @param seconds the max-age, 0 or more + * @throws IllegalArgumentException if seconds is negative + */ + public void setCacheMaxAge(int seconds) { + if (seconds < 0) { + throw new IllegalArgumentException( + "The max-age of the client-side responses cannot be " + + "negative, and was " + seconds + "."); + } + cacheControl = cacheControl(seconds); + } + + private static String cacheControl(int maxAgeSeconds) { + return "private,max-age=" + maxAgeSeconds; + } + private boolean hasControlChar(String str) { for (char chr : str.toCharArray()) { if(chr <= 31) return true; @@ -226,45 +306,61 @@ public void serveContent( HttpServletRequest request, HttpServletResponse response, ContentTypes contentType) throws IOException { - // Get the hash code. - FlowData flowData = flowDataProviderCore.getFlowData(request); - int hash = flowData.generateKey(pipeline.getEvidenceKeyFilter()).hashCode(); - - String ifNoneMatch = request.getHeader("If-None-Match"); - if (ifNoneMatch != null && - Integer.toString(hash).equals(request.getHeader("If-None-Match"))) { - // The response hasn't changed so respond with a 304. - response.setStatus(304); + FlowData flowData = flowDataProviderCore.getFlowData(request); + String etag = getETag(flowData); + + // A response built while processing recorded errors may lack the + // data of the element that failed, so it must not be revalidated + // into a 304 once processing works again. + Collection errors = flowData.getErrors(); + boolean failed = errors != null && errors.isEmpty() == false; + + // A Pipeline without the builder for this endpoint is a + // configuration error, so it is reported before a matching + // If-None-Match could answer 304. + JavaScriptBuilderElement jsElement = null; + JsonBuilderElement jsonElement = null; + switch (contentType) { + case JavaScript: + jsElement = flowData.getPipeline().getElement(JavaScriptBuilderElement.class); + if (jsElement == null) { + throw new PipelineConfigurationException( + "Client-side JavaScript has been requested from " + + "the Pipeline. However, the JavaScriptBuilderElement " + + "is not present. To resolve this error, " + + "either disable client-side evidence or ensure " + + "the JavaScriptBuilderElement is added to " + + "your Pipeline."); + } + break; + case Json: + jsonElement = flowData.getPipeline().getElement(JsonBuilderElement.class); + if (jsonElement == null) { + throw new PipelineConfigurationException( + "JSON data has been requested from the Pipeline. " + + "However, the JsonBuilderElement is not " + + "present. To resolve this error, either " + + "disable client-side evidence or ensure " + + "the JsonBuilderElement is added to your " + + "Pipeline." + ); + } + break; + default: + break; + } + + if (ifNoneMatchApplies(request, etag)) { + answerMatch(request, response, etag); } else { // Otherwise, return the requested content to the client. String content = null; switch (contentType) { case JavaScript: - JavaScriptBuilderElement jsElement = flowData.getPipeline().getElement(JavaScriptBuilderElement.class); - if (jsElement == null) { - throw new PipelineConfigurationException( - "Client-side JavaScript has been requested from " + - "the Pipeline. However, the JavaScriptBuilderElement " + - "is not present. To resolve this error, " + - "either disable client-side evidence or ensure " + - "the JavaScriptBuilderElement is added to " + - "your Pipeline."); - } JavaScriptBuilderData jsData = flowData.getFromElement(jsElement); content = jsData == null ? null : jsData.getJavaScript(); break; case Json: - JsonBuilderElement jsonElement = flowData.getPipeline().getElement(JsonBuilderElement.class); - if (jsonElement == null) { - throw new PipelineConfigurationException( - "JSON data has been requested from the Pipeline. " + - "However, the JsonBuilderElement is not " + - "present. To resolve this error, either " + - "disable client-side evidence or ensure " + - "the JsonBuilderElement is added to your " + - "Pipeline." - ); - } JsonBuilderData jsonData = flowData.getFromElement(jsonElement); content = jsonData == null ? null : jsonData.getJson(); break; @@ -274,35 +370,347 @@ public void serveContent( setHeaders( response, - hash, - content == null ? 0 : content.length(), + etag, + failed, + content, contentType == ContentTypes.JavaScript ? "x-javascript" : "json"); - response.getWriter().write(content); + if (content != null) { + response.getWriter().write(content); + } } } + @Override + public boolean serveJavascriptNotModified( + FlowData flowData, + HttpServletRequest request, + HttpServletResponse response) { + return serveNotModified( + flowData, request, response, JavaScriptBuilderElement.class); + } + + @Override + public boolean serveJsonNotModified( + FlowData flowData, + HttpServletRequest request, + HttpServletResponse response) { + return serveNotModified( + flowData, request, response, JsonBuilderElement.class); + } + + /** + * Answer the request if its If-None-Match matches the ETag of the + * Evidence in the FlowData. A Pipeline without the builder for the + * endpoint is left to be processed, so that the configuration error + * is reported. + */ + private boolean serveNotModified( + FlowData flowData, + HttpServletRequest request, + HttpServletResponse response, + Class> builder) { + if (pipeline == null || + flowData.getPipeline().getElement(builder) == null) { + return false; + } + String etag = getETag(flowData); + if (ifNoneMatchApplies(request, etag) == false) { + return false; + } + answerMatch(request, response, etag); + return true; + } + + /** + * Answer a request whose If-None-Match matches. RFC 9110 section + * 13.1.2 has a GET or HEAD answered with 304 and any other method + * with 412. The ETag depends on the supplied Evidence alone, so the + * 304 carries the headers of a good response, as the 200 it stands + * for would when processing records no errors. + */ + private void answerMatch( + HttpServletRequest request, + HttpServletResponse response, + String etag) { + String method = request.getMethod(); + if ("GET".equals(method) || "HEAD".equals(method)) { + response.setStatus(304); + setCachingHeaders(response, etag, false); + } else { + response.setStatus(412); + response.setHeader("Access-Control-Allow-Origin", "*"); + } + } + + /** + * The ETag of the Evidence the request supplied, a weak entity tag + * holding the SHA-256 digest of the supplied values whose keys are + * in the Pipeline's Evidence key filter. + *

    + * The values are read from the {@link WebRequestEvidence}, which + * holds them as the request supplied them, because Flow Elements add + * Evidence while they process, such as the new session id the + * Sequence Element adds to a request with none. A FlowData whose + * Evidence was added without the web integration has no such + * record, so its Evidence is used instead. + *

    + * Keys are lower cased, as Evidence keys are case insensitive, and + * sorted. Each key and value is written as its length in four bytes, + * then its UTF-8 bytes, so the same values give the same ETag in + * every process. The tag is weak because the content can change for + * the same Evidence, for example when a data file is updated. + * @param flowData the {@link FlowData} of the request + * @return the ETag, such as W/"9bee...dbd3" + */ + private String getETag(FlowData flowData) { + Map evidence = flowData.getEvidence().asKeyMap(); + Object supplied = evidence.get(ShareUsageEvidence.HTTP_REQUEST_KEY); + if (supplied instanceof WebRequestEvidence) { + evidence = (WebRequestEvidence) supplied; + } + + EvidenceKeyFilter filter = pipeline.getEvidenceKeyFilter(); + Map values = new TreeMap<>(); + for (Map.Entry entry : evidence.entrySet()) { + if (entry.getValue() != null && + filter.include(entry.getKey())) { + values.put( + entry.getKey().toLowerCase(Locale.ROOT), + String.valueOf(entry.getValue())); + } + } + + MessageDigest digest; + try { + digest = MessageDigest.getInstance(ETAG_DIGEST); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + for (Map.Entry entry : values.entrySet()) { + addToDigest(digest, entry.getKey()); + addToDigest(digest, entry.getValue()); + } + + byte[] hash = digest.digest(); + StringBuilder etag = new StringBuilder(hash.length * 2 + 4); + etag.append("W/\""); + for (byte b : hash) { + etag.append(HEX[(b >> 4) & 0xF]).append(HEX[b & 0xF]); + } + return etag.append('"').toString(); + } + + /** + * Add the text to the digest as its length in four bytes, big + * endian, then its UTF-8 bytes, so that no two lists of texts give + * the same bytes. + */ + private static void addToDigest(MessageDigest digest, String text) { + byte[] bytes = text.getBytes(StandardCharsets.UTF_8); + digest.update(new byte[] { + (byte) (bytes.length >>> 24), + (byte) (bytes.length >>> 16), + (byte) (bytes.length >>> 8), + (byte) bytes.length }); + digest.update(bytes); + } + + /** + * Whether the request's If-None-Match matches the ETag. RFC 9110 + * section 13.2.1 has a server ignore the conditional headers of a + * method that does not select or change a representation, such as + * CONNECT, OPTIONS or TRACE, so for those it never matches. + * @param request the request + * @param etag the ETag of the response + * @return true if the method must not be performed + */ + private static boolean ifNoneMatchApplies( + HttpServletRequest request, + String etag) { + String method = request.getMethod(); + if ("CONNECT".equals(method) || "OPTIONS".equals(method) || + "TRACE".equals(method)) { + return false; + } + return ifNoneMatchMatches( + request.getHeaders("If-None-Match"), + etag); + } + + /** + * Whether the If-None-Match field matches the ETag, evaluated as RFC + * 9110 section 13.1.2 says. The field lines are one comma separated + * list (section 5.2). "*" matches. A list of entity tags matches when + * one of them has the same quoted part as the ETag, with or without + * W/ on either, which is the weak comparison of section 8.8.3.2. + * Empty list elements and spaces or tabs around commas are allowed + * (sections 5.6.1.2 and 5.6.3). Any other field, such as a tag then + * something that is not a tag, or two tags with no comma between + * them, does not match. + * @param lines the If-None-Match field lines, or null + * @param etag the ETag of the response + * @return true if the method must not be performed + */ + private static boolean ifNoneMatchMatches( + Enumeration lines, + String etag) { + if (lines == null || lines.hasMoreElements() == false) { + return false; + } + StringBuilder field = new StringBuilder(); + boolean first = true; + while (lines.hasMoreElements()) { + String line = lines.nextElement(); + if (first == false) { + field.append(", "); + } + field.append(line == null ? "" : line); + first = false; + } + String value = field.toString(); + if ("*".equals(trimWhitespace(value))) { + return true; + } + + String opaque = etag.substring(etag.indexOf('"')); + boolean matched = false; + boolean afterComma = true; + int index = 0; + while (index < value.length()) { + char c = value.charAt(index); + if (c == ' ' || c == '\t') { + index++; + } else if (c == ',') { + afterComma = true; + index++; + } else if (afterComma) { + int start = value.startsWith("W/", index) ? + index + 2 : + index; + int end = endOfOpaqueTag(value, start); + if (end < 0) { + return false; + } + if (opaque.equals(value.substring(start, end))) { + matched = true; + } + afterComma = false; + index = end; + } else { + return false; + } + } + return matched; + } + + /** + * The index after the opaque tag starting at start, a double quote, + * any characters an entity tag allows, and a double quote, or -1 if + * there is none (RFC 9110 section 8.8.3). + */ + private static int endOfOpaqueTag(String value, int start) { + if (start >= value.length() || value.charAt(start) != '"') { + return -1; + } + for (int index = start + 1; index < value.length(); index++) { + char c = value.charAt(index); + if (c == '"') { + return index + 1; + } + // etagc is %x21 / %x23-7E / obs-text (%x80-FF). + if ((c == 0x21 || + (c >= 0x23 && c <= 0x7E) || + (c >= 0x80 && c <= 0xFF)) == false) { + return -1; + } + } + return -1; + } + + /** + * The text without leading and trailing spaces and tabs. + */ + private static String trimWhitespace(String text) { + int start = 0; + int end = text.length(); + while (start < end && + (text.charAt(start) == ' ' || text.charAt(start) == '\t')) { + start++; + } + while (end > start && + (text.charAt(end - 1) == ' ' || text.charAt(end - 1) == '\t')) { + end--; + } + return text.substring(start, end); + } + /** * Set various HTTP headers on the JavaScript response. * @param response the {@link HttpServletResponse} to add the response * headers to - * @param hash the hash to use for the ETag - * @param contentLength the length of the returned content. This is used - * for the 'Content-Length' header + * @param etag the ETag + * @param failed true when processing recorded errors, which sends + * no-cache in place of max-age, and no ETag + * @param content the content, or null if there is none. Its length + * in bytes, in the character encoding the response is + * written with, is the 'Content-Length' header * @param contentType the type of content */ private void setHeaders( HttpServletResponse response, - int hash, - int contentLength, + String etag, + boolean failed, + String content, String contentType) { response.setContentType("application/" + contentType); - response.setContentLength(contentLength); + response.setContentLength(content == null ? + 0 : + content.getBytes(characterEncoding(response)).length); response.setStatus(200); - response.setHeader("Cache-Control", stringJoin(cacheControl, ",")); - response.setHeader("Vary", stringJoin(getHeadersAffectingJavaScript(), ",")); - response.setHeader("ETag", Integer.toString(hash)); + setCachingHeaders(response, etag, failed); + } + + /** + * The character encoding the response's writer uses, which is + * ISO-8859-1 when none has been set, as the servlet specification + * says. + */ + private static Charset characterEncoding( + HttpServletResponse response) { + String name = response.getCharacterEncoding(); + return name == null ? + StandardCharsets.ISO_8859_1 : + Charset.forName(name); + } + + /** + * Set the headers a 200 and the 304 that stands for it both carry, + * which are Vary, Cache-Control, the ETag unless processing recorded + * errors, and the CORS header. + * @param response the {@link HttpServletResponse} to add the response + * headers to + * @param etag the ETag + * @param failed true when processing recorded errors, which sends + * no-cache in place of max-age, and no ETag + */ + private void setCachingHeaders( + HttpServletResponse response, + String etag, + boolean failed) { + // Where no header affects the response there is no Vary header, + // rather than an empty one. + List vary = getHeadersAffectingJavaScript(); + if (vary.isEmpty() == false) { + response.setHeader("Vary", stringJoin(vary, ",")); + } + if (failed) { + response.setHeader("Cache-Control", FAILED_CACHE_CONTROL); + } else { + response.setHeader("Cache-Control", cacheControl); + response.setHeader("ETag", etag); + } response.setHeader("Access-Control-Allow-Origin", "*"); } } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/FiftyOneJSServiceCore.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/FiftyOneJSServiceCore.java index 70d27fa0b..7ec017007 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/FiftyOneJSServiceCore.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/FiftyOneJSServiceCore.java @@ -66,6 +66,29 @@ boolean serveJS(HttpServletRequest request, HttpServletResponse response) boolean serveJson(HttpServletRequest request, HttpServletResponse response) throws IOException; + /** + * Answer a request for the 51Degrees JavaScript or JSON whose + * If-None-Match matches the ETag of the Evidence it supplies, without + * processing it, because that Evidence alone decides the content. A GET + * or HEAD gets a 304 and any other method a 412. Call it before the + * request is processed. + * @param request the {@link HttpServletRequest} to answer + * @param response the {@link HttpServletResponse} to write the answer to + * @param resultService the service that creates the request's + * {@link FlowData}, with the Evidence processing + * would use + * @return true if the request was answered and is not to be processed, + * false otherwise + * @throws IOException if there was a failure reading or writing to the + * request or response + */ + default boolean serveNotModified( + HttpServletRequest request, + HttpServletResponse response, + PipelineResultServiceCore resultService) throws IOException { + return false; + } + /** * Default implementation of the {@link FiftyOneJSServiceCore} service. */ @@ -192,6 +215,38 @@ public boolean serveJson( return result; } + @Override + public boolean serveNotModified( + HttpServletRequest request, + HttpServletResponse response, + PipelineResultServiceCore resultService) throws IOException { + // Most requests carry no If-None-Match, and need nothing more. + if (enabled == false || + request.getHeader("If-None-Match") == null) { + return false; + } + boolean javaScript = matches(request, javaScriptPath); + if (javaScript == false && + matches(request, jsonPath) == false && + matches(request, CORE_JSON_NAME) == false) { + return false; + } + try (FlowData flowData = resultService.createFlowData(request)) { + if (flowData == null) { + return false; + } + return javaScript ? + clientsidePropertyServiceCore.serveJavascriptNotModified( + flowData, request, response) : + clientsidePropertyServiceCore.serveJsonNotModified( + flowData, request, response); + } catch (IOException | RuntimeException e) { + throw e; + } catch (Exception e) { + throw new IOException("FlowData could not be disposed of.", e); + } + } + private void serveCoreJson( HttpServletRequest request, HttpServletResponse response) throws IOException { diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/FlowDataProviderCore.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/FlowDataProviderCore.java index c24f95498..11b42b490 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/FlowDataProviderCore.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/FlowDataProviderCore.java @@ -31,6 +31,11 @@ /** * Provider to be injected into controllers to get the pre-processed * {@link FlowData} for the request being serviced. + *

    + * A Flow Data cannot be used by two threads at once. Application code reads + * the values it needs from the Flow Data on the request's thread, and hands + * those values, not the Flow Data, to any other thread or task. The Flow + * Data is closed when the request ends. */ public interface FlowDataProviderCore { diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/PipelineResultServiceCore.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/PipelineResultServiceCore.java index 289c90ff9..c6692a172 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/PipelineResultServiceCore.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/PipelineResultServiceCore.java @@ -42,6 +42,17 @@ public interface PipelineResultServiceCore { */ void process(HttpServletRequest request); + /** + * Create a FlowData holding the Evidence from the request, as + * {@link #process(HttpServletRequest)} would, without processing it. The + * caller closes it. + * @param request the {@link HttpServletRequest} to take the Evidence from + * @return the FlowData, or null if this service cannot create one + */ + default FlowData createFlowData(HttpServletRequest request) { + return null; + } + /** * Default implementation of the {@link PipelineResultServiceCore} service. */ @@ -79,23 +90,38 @@ public Default( @Override @SuppressWarnings("PMD.FlowDataInTryWithResources") public void process(HttpServletRequest request) { - // Create the flowData - FlowData flowData = pipeline.createFlowData(); + FlowData flowData = createFlowData(request); try { - // Extract the required pieces of evidence from the request - evidenceService.addEvidenceFromRequest(flowData, request); // Start processing the data flowData.process(); } catch (RuntimeException | Error e) { - try { - flowData.close(); - } catch (Exception closeException) { - e.addSuppressed(closeException); - } + closeAfterFailure(flowData, e); throw e; } // Store the FlowData in the request request.setAttribute(HTTPCONTEXT_FLOWDATA, flowData); } + + @Override + @SuppressWarnings("PMD.FlowDataInTryWithResources") + public FlowData createFlowData(HttpServletRequest request) { + FlowData flowData = pipeline.createFlowData(); + try { + // Extract the required pieces of evidence from the request + evidenceService.addEvidenceFromRequest(flowData, request); + } catch (RuntimeException | Error e) { + closeAfterFailure(flowData, e); + throw e; + } + return flowData; + } + + private static void closeAfterFailure(FlowData flowData, Throwable e) { + try { + flowData.close(); + } catch (Exception closeException) { + e.addSuppressed(closeException); + } + } } } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidence.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidence.java index 6206d41e5..ff0152154 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidence.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidence.java @@ -36,7 +36,9 @@ * for a request, accepted or not, keyed the way Evidence is keyed, such as * header.user-agent. It is added to Evidence as * {@link ShareUsageEvidence#HTTP_REQUEST_KEY} for the usage sharing element, - * which accepts no Evidence and reads the request here instead. + * which accepts no Evidence and reads the request here instead. The + * client-side endpoints make their ETag from it too, because it holds the + * values as the request supplied them, before any Flow Element adds its own. *

    * It is a map rather than the HttpServletRequest because the usage sharing * element does not depend on the servlet API, and the values are copied so diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java index 03f853e17..c9f69c96e 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java @@ -65,6 +65,9 @@ class Default implements WebRequestEvidenceServiceCore { private static final String EVIDENCE_REQUEST_PATH = EVIDENCE_SERVER_PREFIX + EVIDENCE_SEPERATOR + "request-path"; + private static final String EVIDENCE_REQUEST_QUERY = + EVIDENCE_SERVER_PREFIX + EVIDENCE_SEPERATOR + "request-query"; + @Override public void addEvidenceFromRequest( FlowData flowData, @@ -127,12 +130,20 @@ public void addEvidenceFromRequest( checkAndAdd(flowData, EVIDENCE_PROTOCOL, request.getScheme()); - // The method and the path of the request line, as sent. + // The method, the path and the query string of the request line, + // as sent. The query string is empty when the request has none, + // so an absent key means only that the request line is not + // supplied. checkAndAdd(flowData, EVIDENCE_REQUEST_METHOD, request.getMethod()); checkAndAdd( flowData, EVIDENCE_REQUEST_PATH, request.getRequestURI()); + String query = request.getQueryString(); + checkAndAdd( + flowData, + EVIDENCE_REQUEST_QUERY, + query == null ? "" : query); } /** diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/AddedElementPositionTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/AddedElementPositionTests.java new file mode 100644 index 000000000..0c8734ba5 --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/AddedElementPositionTests.java @@ -0,0 +1,158 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web; + +import fiftyone.pipeline.core.configuration.ElementOptions; +import fiftyone.pipeline.core.configuration.PipelineOptions; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement; +import fiftyone.pipeline.engines.fiftyone.flowelements.SetHeadersElement; +import fiftyone.pipeline.engines.fiftyone.flowelements.ShareUsageElement; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElement; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * The positions the web integration gives the elements it adds, as the web + * integration specification's Pipeline construction section sets them out. + * The Sequence Element is added first, the JavaScript Builder Element after + * every other element, the JSON Builder Element immediately before the + * JavaScript Builder Element, and the Set Headers Element after every other + * element, so the last two are the JavaScript Builder and the Set Headers + * Element in either order. + */ +public class AddedElementPositionTests { + + private static final String MULTIPLY_BY = "MultiplyByElement"; + + private static final String SHARE_USAGE_BUILDER = "ShareUsage"; + + private static final String SHARE_USAGE = + ShareUsageElement.class.getSimpleName(); + + private static final String JAVASCRIPT_BUILDER = + JavaScriptBuilderElement.class.getSimpleName(); + + private static ElementOptions element(String builderName) { + ElementOptions element = new ElementOptions(); + element.builderName = builderName; + return element; + } + + private static ElementOptions multiplyBy() { + ElementOptions element = element(MULTIPLY_BY); + element.buildParameters.put("Multiple", "8"); + return element; + } + + /** + * Builds the Pipeline as the servlet filter does, with client-side + * properties, the Sequence Element and the Set Headers Element turned on, + * and returns the simple class names of its elements in order. + */ + private static List namesBuiltFrom(ElementOptions... configured) + throws Exception { + PipelineOptions options = new PipelineOptions(); + for (ElementOptions element : configured) { + options.elements.add(element); + } + List names = new ArrayList<>(); + try (Pipeline pipeline = StartupHelpers.buildFromConfiguration( + new PipelineBuilder(), options, true, null, + Constants.DEFAULT_JSON_PATH, true, true)) { + for (FlowElement element : pipeline.getFlowElements()) { + names.add(element.getClass().getSimpleName()); + } + } + return names; + } + + private static void assertPositions(List names) { + int last = names.size() - 1; + int sequence = names.indexOf(SequenceElement.class.getSimpleName()); + int json = names.indexOf(JsonBuilderElement.class.getSimpleName()); + int javaScript = names.indexOf(JAVASCRIPT_BUILDER); + int setHeaders = names.indexOf(SetHeadersElement.class.getSimpleName()); + assertEquals("The Sequence Element should be first. " + names, + 0, sequence); + assertEquals("The JSON Builder Element should be immediately before " + + "the JavaScript Builder Element. " + names, + javaScript - 1, json); + assertTrue("The JavaScript Builder and Set Headers Elements should " + + "be the last two. " + names, + Math.min(javaScript, setHeaders) == last - 1 && + Math.max(javaScript, setHeaders) == last); + } + + /** + * With only an element of the application's configured, each added + * element takes its position around it. + */ + @Test + public void AddedElements_TakeTheirPositions() throws Exception { + List names = namesBuiltFrom(multiplyBy()); + + assertEquals(names.toString(), 5, names.size()); + assertEquals(names.toString(), 1, names.indexOf(MULTIPLY_BY)); + assertPositions(names); + } + + /** + * With a JavaScript Builder Element configured, the JSON Builder Element + * is added immediately before it. + */ + @Test + public void JsonBuilder_AddedImmediatelyBeforeAConfiguredJavaScriptBuilder() + throws Exception { + List names = namesBuiltFrom( + multiplyBy(), element(JAVASCRIPT_BUILDER)); + + assertEquals(names.toString(), 5, names.size()); + assertPositions(names); + } + + /** + * With the usage sharing element configured last, each added element + * goes before it, so usage sharing stays the last Flow Element, as it + * must be. + */ + @Test + public void AddedElements_GoBeforeAConfiguredUsageSharingElement() + throws Exception { + List names = namesBuiltFrom( + multiplyBy(), element(SHARE_USAGE_BUILDER)); + + assertEquals(names.toString(), 6, names.size()); + assertEquals("The usage sharing element should be last. " + names, + names.size() - 1, names.indexOf(SHARE_USAGE)); + assertPositions(names.subList(0, names.size() - 1)); + } +} diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsideJsonEndpointTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsideJsonEndpointTests.java new file mode 100644 index 000000000..082ddf9ac --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsideJsonEndpointTests.java @@ -0,0 +1,212 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web; + +import fiftyone.pipeline.core.data.ElementDataBase; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.ElementPropertyMetaDataDefault; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.FlowElementBase; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; +import fiftyone.pipeline.web.services.ClientsidePropertyServiceCore; +import fiftyone.pipeline.web.services.FlowDataProviderCore; +import fiftyone.pipeline.web.services.WebRequestEvidenceServiceCore; +import org.json.JSONObject; +import org.json.JSONTokener; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The JSON endpoint answers with JSON in the same form as the JSON the + * JavaScript holds, as the SetEndpoint row of the JavaScript Builder's + * configuration options asks. + * @see Specification + */ +public class ClientsideJsonEndpointTests { + + private static final ILoggerFactory LOGGERS = + LoggerFactory.getILoggerFactory(); + + /** + * The text that the template puts before the JSON in the script, with + * or without the white space that minifying takes out. + */ + private static final Pattern JSON_IN_SCRIPT = + Pattern.compile("var json\\s*=\\s*"); + + /** + * The JSON served by the JSON endpoint and the JSON the script served by + * the JavaScript endpoint holds, for one request, are the same, with a + * value, a Property assigned no value and its nullreason, and the + * JavaScript Property and session values the builders add. + */ + @Test + public void JsonEndpoint_SameFormAsJsonInJavaScript() throws Exception { + try (Pipeline pipeline = new PipelineBuilder(LOGGERS) + .addFlowElement(new SequenceElementBuilder(LOGGERS).build()) + .addFlowElement(new ValuesElement()) + .addFlowElement(new JsonBuilderElementBuilder(LOGGERS).build()) + .addFlowElement(new JavaScriptBuilderElementBuilder(LOGGERS) + .setEndpoint("/51dpipeline/json") + .build()) + .setAutoCloseElements(true) + .build(); + FlowData data = pipeline.createFlowData()) { + HttpServletRequest request = request(); + new WebRequestEvidenceServiceCore.Default() + .addEvidenceFromRequest(data, request); + data.process(); + assertTrue("Processing recorded errors: " + data.getErrors(), + data.getErrors().isEmpty()); + + FlowDataProviderCore provider = mock(FlowDataProviderCore.class); + when(provider.getFlowData(request)).thenReturn(data); + ClientsidePropertyServiceCore service = + new ClientsidePropertyServiceCore.Default(provider, pipeline); + + StringWriter json = new StringWriter(); + service.serveJson(request, response(json)); + StringWriter javaScript = new StringWriter(); + service.serveJavascript(request, response(javaScript)); + + JSONObject served = new JSONObject(json.toString()); + JSONObject embedded = jsonInScript(javaScript.toString()); + assertTrue("The JSON endpoint answered\n" + served.toString(2) + + "\nbut the script holds\n" + embedded.toString(2), + served.similar(embedded)); + JSONObject values = served.getJSONObject("values"); + assertEquals("a value", values.getString("value")); + assertTrue(values.isNull("novalue")); + assertEquals("No value for the test.", + values.getString("novaluenullreason")); + } + } + + /** + * The JSON object the script assigns to its json variable. + */ + private static JSONObject jsonInScript(String javaScript) { + Matcher matcher = JSON_IN_SCRIPT.matcher(javaScript); + assertTrue("The script holds no JSON: " + javaScript, matcher.find()); + return (JSONObject) new JSONTokener( + javaScript.substring(matcher.end())) + .nextValue(); + } + + private static HttpServletRequest request() { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getMethod()).thenReturn("GET"); + when(request.getRequestURI()).thenReturn("/51dpipeline/json"); + when(request.getHeaderNames()).thenReturn( + Collections.enumeration(Arrays.asList("Host"))); + when(request.getHeader("Host")).thenReturn("example.com"); + when(request.getRemoteAddr()).thenReturn("1.2.3.4"); + when(request.getScheme()).thenReturn("https"); + return request; + } + + private static HttpServletResponse response(StringWriter body) + throws Exception { + HttpServletResponse response = mock(HttpServletResponse.class); + when(response.getWriter()).thenReturn(new PrintWriter(body, true)); + return response; + } + + /** + * An element that adds a value, a Property assigned no value and a + * JavaScript Property, in place of an engine. + */ + private static class ValuesElement + extends FlowElementBase { + + ValuesElement() { + super(LoggerFactory.getLogger(ValuesElement.class), null); + } + + @Override + protected void processInternal(FlowData data) { + ElementDataBase values = + new ElementDataBase(logger, data) { }; + values.put("value", "a value"); + AspectPropertyValueDefault noValue = + new AspectPropertyValueDefault<>(); + noValue.setNoValueMessage("No value for the test."); + values.put("novalue", noValue); + values.put("javascript", "var a = 1;"); + data.getOrAdd(getElementDataKey(), d -> values); + } + + @Override + public String getElementDataKey() { + return "values"; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist( + Collections.emptyList()); + } + + @Override + public List getProperties() { + return Arrays.asList( + new ElementPropertyMetaDataDefault( + "value", this, "test", String.class, true), + new ElementPropertyMetaDataDefault( + "novalue", this, "test", String.class, true), + new ElementPropertyMetaDataDefault( + "javascript", this, "test", String.class, true)); + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + } +} diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsidePropertyServiceTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsidePropertyServiceTests.java index 31c8f9d46..32a80f259 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsidePropertyServiceTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsidePropertyServiceTests.java @@ -24,23 +24,39 @@ import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatcher; +import org.slf4j.LoggerFactory; -import fiftyone.pipeline.core.data.DataKey; +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.Evidence; import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterAggregator; import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.FlowElementBase; import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; import fiftyone.pipeline.javascriptbuilder.data.JavaScriptBuilderData; import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; import fiftyone.pipeline.jsonbuilder.data.JsonBuilderData; import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElement; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; import fiftyone.pipeline.web.services.ClientsidePropertyServiceCore; import fiftyone.pipeline.web.services.FlowDataProviderCore; +import fiftyone.pipeline.web.services.WebRequestEvidenceServiceCore; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @@ -49,7 +65,10 @@ import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -60,7 +79,7 @@ public class ClientsidePropertyServiceTests { private FlowDataProviderCore flowDataProvider; private Pipeline pipeline; private FlowData flowData; - private DataKey dataKey; + private Map evidence; private JavaScriptBuilderElement jsElement; private JsonBuilderElement jsonElement; private JavaScriptBuilderData jsData; @@ -75,9 +94,35 @@ public class ClientsidePropertyServiceTests { private static final String JS_CONTENT_TYPE = "application/x-javascript"; private static final String JSON_CONTENT_TYPE = "application/json"; + private static final String USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"; + + /** + * A weak entity tag holding a quoted SHA-256 digest in lower case hex. + */ + private static final String WEAK_ETAG = "W/\"[0-9a-f]{64}\""; + + /** + * The SHA-256 digest of the keys and values below, sorted by key, each + * written as its length in four bytes, big endian, then its UTF-8 bytes, + * worked out apart from the Java code. + * header.sec-ch-ua-mobile = ?0 + * header.user-agent = USER_AGENT + */ + private static final String ETAG_UA_AND_MOBILE = + "W/\"9beee8189f1e05e5f821379597c3a690e612c465dc53d67de2e42ebffbd4dbd3\""; + + /** + * As above, for header.user-agent = USER_AGENT alone. + */ + private static final String ETAG_UA = + "W/\"4eee715bd966ea5198b7d696aea05fd627dfd2df7f8be928e95797155ec2cf00\""; + @Before public void init() throws IOException { request = mock(HttpServletRequest.class); + when(request.getMethod()).thenReturn("GET"); response = mock(HttpServletResponse.class); responseString = new StringWriter(); PrintWriter responseWriter = new PrintWriter(responseString); @@ -87,8 +132,10 @@ public void init() throws IOException { jsonElement = mock(JsonBuilderElement.class); flowData = mock(FlowData.class); - dataKey = mock(DataKey.class); - when(flowData.generateKey(any(EvidenceKeyFilter.class))).thenReturn(dataKey); + evidence = new HashMap<>(); + Evidence flowEvidence = mock(Evidence.class); + when(flowEvidence.asKeyMap()).thenReturn(evidence); + when(flowData.getEvidence()).thenReturn(flowEvidence); jsData = mock(JavaScriptBuilderData.class); when(jsData.getJavaScript()).thenReturn(JS_CONTENT); when(flowData.getFromElement(jsElement)).thenReturn(jsData); @@ -148,6 +195,10 @@ public void JavaScript_VaryHeader_UA_AND_SEC() throws IOException { JavaScript_VaryHeader(elements, "User-Agent,Sec-ch-ua"); } + /** + * An Evidence key filter that names no real header gives no Vary header, + * rather than an empty one. + */ @Test public void JavaScript_VaryHeader_PSEUDO() throws IOException { List> elements = new ArrayList>(); @@ -155,7 +206,7 @@ public void JavaScript_VaryHeader_PSEUDO() throws IOException { "header.sec-ch-uasec-ch-ua-full-version"); elements.add(element); - JavaScript_VaryHeader(elements, ""); + JavaScript_VaryHeader(elements, null); } @Test @@ -208,8 +259,799 @@ public void JavaScript_VaryHeader_2Element_UA_AND_SEC() throws IOException { JavaScript_VaryHeader(elements, "User-Agent,Sec-ch-ua"); } + /** + * The headers of the elements in a parallel group are in the Pipeline's + * Evidence key filter, so they are in the Vary header too. + */ + @Test + @SuppressWarnings("rawtypes") + public void JavaScript_VaryHeader_ParallelElements() throws IOException { + EvidenceKeyFilterAggregator group = new EvidenceKeyFilterAggregator(); + group.addFilter(new EvidenceKeyFilterWhitelist( + Arrays.asList("header.User-Agent"), + String.CASE_INSENSITIVE_ORDER)); + FlowElement element = mock(FlowElement.class); + when(element.getEvidenceKeyFilter()).thenReturn(group); + List elements = new ArrayList<>(); + elements.add(element); + Configure(elements); + + service.serveJavascript(request, response); + + verify(response).setHeader(eq("Vary"), eq("User-Agent")); + } + + /** + * The ETag is a weak entity tag, because the content can change for the + * same Evidence, for example when a data file is updated, and RFC 9110 + * section 8.8.3 asks for such a validator to be marked weak. Its opaque + * part is a quoted hash. + */ + @Test + public void JavaScript_ETag_IsWeakEntityTag() throws IOException { + Configure(null); + + service.serveJavascript(request, response); + + verify(response).setHeader(eq("ETag"), matches(WEAK_ETAG)); + } + + @Test + public void Json_ETag_IsWeakEntityTag() throws IOException { + Configure(null); + + service.serveJson(request, response); + + verify(response).setHeader(eq("ETag"), matches(WEAK_ETAG)); + } + + /** + * The same Evidence gives the same ETag in every process, so the ETag is + * a digest with no per-process seed, compared here with a value worked + * out apart from the Java code. Evidence outside the Pipeline's Evidence + * key filter, and the case of a key, leave it unchanged. + */ + @Test + public void JavaScript_ETag_SameInEveryProcess() throws IOException { + evidence.put("header.User-Agent", USER_AGENT); + evidence.put("header.sec-ch-ua-mobile", "?0"); + evidence.put("header.X-Not-Accepted", "anything"); + Configure(null); + when(pipeline.getEvidenceKeyFilter()).thenReturn( + new EvidenceKeyFilterWhitelist( + Arrays.asList("header.user-agent", "header.sec-ch-ua-mobile"), + String.CASE_INSENSITIVE_ORDER)); + + service.serveJavascript(request, response); + + verify(response).setHeader(eq("ETag"), eq(ETAG_UA_AND_MOBILE)); + } + + /** + * Two Pipelines built the same way, as in two processes, give the same + * ETag for the same request. + */ + @Test + public void Json_ETag_SameAcrossPipelines() throws Exception { + try (Pipeline first = webPipeline(); + Pipeline second = webPipeline(); + FlowData firstData = first.createFlowData(); + FlowData secondData = second.createFlowData()) { + String firstETag = + servedETag(first, firstData, webRequest(USER_AGENT)); + String secondETag = + servedETag(second, secondData, webRequest(USER_AGENT)); + + assertEquals("Two Pipelines gave different ETags.", + firstETag, secondETag); + assertEquals(ETAG_UA, firstETag); + } + } + + /** + * The Sequence Element adds a new session id to every request that has + * none, so the ETag is made from the Evidence as the request supplied it, + * and two requests with the same Evidence get the same ETag. A different + * User-Agent gives a different one. + */ + @Test + public void Json_ETag_UnchangedWhenSequenceElementAddsSessionId() + throws Exception { + try (Pipeline webPipeline = webPipeline(); + FlowData firstData = webPipeline.createFlowData(); + FlowData secondData = webPipeline.createFlowData(); + FlowData otherData = webPipeline.createFlowData()) { + String firstETag = + servedETag(webPipeline, firstData, webRequest(USER_AGENT)); + String secondETag = + servedETag(webPipeline, secondData, webRequest(USER_AGENT)); + String otherETag = + servedETag(webPipeline, otherData, webRequest("Other")); + + Object firstSession = + firstData.getEvidence().asKeyMap().get("query.session-id"); + Object secondSession = + secondData.getEvidence().asKeyMap().get("query.session-id"); + assertNotNull("The Sequence Element added no session id.", + firstSession); + assertNotEquals("The Sequence Element added the same session id.", + firstSession, secondSession); + assertEquals("The ETag changed with the session id.", + firstETag, secondETag); + assertNotEquals("The ETag did not change with the User-Agent.", + firstETag, otherETag); + } + } + + /** + * A response built while processing recorded errors may lack the data of + * the element that failed, so it is not kept and carries no ETag to + * revalidate with. + */ + @Test + public void JavaScript_ProcessingErrors_NoCacheAndNoETag() + throws IOException { + recordProcessingError(); + Configure(null); + + service.serveJavascript(request, response); + + ValidateFailedResponse(JS_CONTENT, JS_CONTENT_TYPE); + } + + @Test + public void Json_ProcessingErrors_NoCacheAndNoETag() throws IOException { + recordProcessingError(); + Configure(null); + + service.serveJson(request, response); + + ValidateFailedResponse(JSON_CONTENT, JSON_CONTENT_TYPE); + } + + /** + * The ETag depends on the Evidence alone, so a matching If-None-Match gets + * the same 304 while processing fails as at any other time, carrying the + * headers of a good response. A browser holding a good response keeps it, + * with its ETag and max-age. + */ + @Test + public void JavaScript_ProcessingErrors_MatchingIfNoneMatch_304() + throws IOException { + Configure(ConfigureElements( + Collections.singletonList(Arrays.asList("header.User-Agent")))); + String etag = etagSent(); + recordProcessingError(); + ifNoneMatch(etag); + + service.serveJavascript(request, response); + + Validate304(etag); + } + + /** + * A matching If-None-Match gets the same status while processing fails as + * at any other time, so a POST gets 412. + */ + @Test + public void Json_ProcessingErrors_MatchingIfNoneMatch_Post_412() + throws IOException { + Configure(null); + String etag = etagSent(); + recordProcessingError(); + when(request.getMethod()).thenReturn("POST"); + ifNoneMatch(etag); + + service.serveJson(request, response); + + verify(response).setStatus(eq(412)); + verify(response, times(0)).setHeader(eq("ETag"), anyString()); + assertEquals("", responseString.toString()); + } + + /** + * A 304 carries the Cache-Control, Vary and ETag headers a 200 to the same + * request would, as RFC 9110 section 15.4.5 requires, and the CORS header. + */ + @Test + public void JavaScript_MatchingIfNoneMatch_304_CarriesHeaders() + throws IOException { + Configure(ConfigureElements( + Collections.singletonList(Arrays.asList("header.User-Agent")))); + String etag = etagSent(); + ifNoneMatch(etag); + + service.serveJavascript(request, response); + + Validate304(etag); + } + + @Test + public void Json_MatchingIfNoneMatch_Get_304_CarriesHeaders() + throws IOException { + Configure(ConfigureElements( + Collections.singletonList(Arrays.asList("header.User-Agent")))); + String etag = etagSent(); + ifNoneMatch(etag); + + service.serveJson(request, response); + + Validate304(etag); + } + + /** + * A Pipeline without the JavaScript builder fails a request for the + * JavaScript with a configuration error even when its If-None-Match + * matches, rather than answering 304. + */ + @Test + public void JavaScript_MissingBuilder_MatchingIfNoneMatch_Throws() + throws IOException { + Configure(ConfigureElements( + Collections.singletonList(Arrays.asList("header.User-Agent")))); + String etag = etagSent(); + ifNoneMatch(etag); + when(pipeline.getElement(JavaScriptBuilderElement.class)).thenReturn(null); + + try { + service.serveJavascript(request, response); + fail("A PipelineConfigurationException should have been thrown"); + } catch (PipelineConfigurationException e) { + verify(response, times(0)).setStatus(eq(304)); + } + } + + /** + * As above, for the JSON endpoint without the JSON builder. + */ + @Test + public void Json_MissingBuilder_MatchingIfNoneMatch_Throws() + throws IOException { + Configure(ConfigureElements( + Collections.singletonList(Arrays.asList("header.User-Agent")))); + String etag = etagSent(); + ifNoneMatch(etag); + when(pipeline.getElement(JsonBuilderElement.class)).thenReturn(null); + + try { + service.serveJson(request, response); + fail("A PipelineConfigurationException should have been thrown"); + } catch (PipelineConfigurationException e) { + verify(response, times(0)).setStatus(eq(304)); + } + } + + @Test + public void JavaScript_MatchingIfNoneMatch_Head_304() throws IOException { + Configure(null); + String etag = etagSent(); + when(request.getMethod()).thenReturn("HEAD"); + ifNoneMatch(etag); + + service.serveJavascript(request, response); + + verify(response).setStatus(eq(304)); + assertEquals("", responseString.toString()); + } + + /** + * RFC 9110 section 13.1.2 answers a matching If-None-Match with 412 for + * any method other than GET or HEAD, such as the POST the client-side + * script sends to the JSON endpoint. + */ + @Test + public void Json_MatchingIfNoneMatch_Post_412() throws IOException { + Configure(null); + String etag = etagSent(); + when(request.getMethod()).thenReturn("POST"); + ifNoneMatch(etag); + + service.serveJson(request, response); + + verify(response).setStatus(eq(412)); + verify(response, times(0)).setStatus(eq(304)); + verify(response).setHeader(eq("Access-Control-Allow-Origin"), eq("*")); + assertEquals("", responseString.toString()); + } + + @Test + public void JavaScript_MatchingIfNoneMatch_Post_412() throws IOException { + Configure(null); + String etag = etagSent(); + when(request.getMethod()).thenReturn("POST"); + ifNoneMatch(etag); + + service.serveJavascript(request, response); + + verify(response).setStatus(eq(412)); + assertEquals("", responseString.toString()); + } + + /** + * RFC 9110 section 13.2.1 has a server ignore If-None-Match on a method + * that does not select or change a representation, such as OPTIONS, so + * a matching one is processed and answered as without the header. + */ + @Test + public void Json_MatchingIfNoneMatch_Options_Ignored() throws IOException { + Configure(null); + String etag = etagSent(); + when(request.getMethod()).thenReturn("OPTIONS"); + ifNoneMatch(etag); + + service.serveJson(request, response); + + verify(response, times(0)).setStatus(eq(412)); + verify(response, times(0)).setStatus(eq(304)); + verify(response).setStatus(eq(200)); + } + + /** + * Before processing, If-None-Match on a TRACE, an OPTIONS or a CONNECT is + * ignored, so the request is left to be processed. + */ + @Test + public void NotModified_IgnoredMethods_NotAnswered() throws IOException { + Configure(null); + for (String method : new String[] {"CONNECT", "OPTIONS", "TRACE"}) { + when(request.getMethod()).thenReturn(method); + ifNoneMatch("*"); + + assertFalse(method, + service.serveJavascriptNotModified(flowData, request, response)); + assertFalse(method, + service.serveJsonNotModified(flowData, request, response)); + verify(response, times(0)).setStatus(anyInt()); + } + } + + /** + * Without a matching If-None-Match a POST gets the JSON as before. + */ + @Test + public void Json_Post_200() throws IOException { + Configure(null); + when(request.getMethod()).thenReturn("POST"); + + service.serveJson(request, response); + + verify(response).setStatus(eq(200)); + assertEquals(JSON_CONTENT, responseString.toString()); + } + + /** + * The max-age can be set, as the specification says it SHOULD be, and a + * 200 and a 304 both carry it. + */ + @Test + public void JavaScript_CacheMaxAge_Configured() throws IOException { + Configure(null); + ((ClientsidePropertyServiceCore.Default) service).setCacheMaxAge(60); + + service.serveJavascript(request, response); + + verify(response).setHeader(eq("Cache-Control"), eq("private,max-age=60")); + } + + @Test + public void Json_CacheMaxAge_Configured_304() throws IOException { + Configure(null); + ((ClientsidePropertyServiceCore.Default) service).setCacheMaxAge(0); + ifNoneMatch(etagSent()); + + service.serveJson(request, response); + + verify(response).setStatus(eq(304)); + verify(response).setHeader(eq("Cache-Control"), eq("private,max-age=0")); + } + + /** + * A failed response is not kept, whatever the max-age. + */ + @Test + public void JavaScript_CacheMaxAge_FailedResponseNoCache() + throws IOException { + recordProcessingError(); + Configure(null); + ((ClientsidePropertyServiceCore.Default) service).setCacheMaxAge(60); + + service.serveJavascript(request, response); + + ValidateFailedResponse(JS_CONTENT, JS_CONTENT_TYPE); + } + + @Test(expected = IllegalArgumentException.class) + public void CacheMaxAge_Negative_Refused() { + Configure(null); + ((ClientsidePropertyServiceCore.Default) service).setCacheMaxAge(-1); + } + + /** + * "*" matches, as the endpoint always has a representation. RFC 9110 + * section 13.1.2 step 1. + */ + @Test + public void IfNoneMatch_Star_304() throws IOException { + Configure(null); + validateIfNoneMatch(304, " * "); + } + + /** + * A list matches when one of its tags matches. Step 2. + */ + @Test + public void IfNoneMatch_ListHoldingTheTag_304() throws IOException { + Configure(null); + validateIfNoneMatch(304, "W/\"0\", " + etagSent() + ", \"1\""); + } + + /** + * The comparison is weak, so the tag matches without W/ as well. + * RFC 9110 section 8.8.3.2. + */ + @Test + public void IfNoneMatch_TagWithoutWeakPrefix_304() throws IOException { + Configure(null); + validateIfNoneMatch(304, etagSent().substring(2)); + } + + /** + * List elements may be separated by tabs as well as spaces, and empty + * elements are ignored. RFC 9110 sections 5.6.1.2 and 5.6.3. + */ + @Test + public void IfNoneMatch_TabBetweenElements_304() throws IOException { + Configure(null); + validateIfNoneMatch(304, ",W/\"0\",\t" + etagSent() + "\t, ,"); + } + + /** + * Several field lines are one list. RFC 9110 section 5.2. + */ + @Test + public void IfNoneMatch_SeveralFieldLines_304() throws IOException { + Configure(null); + validateIfNoneMatch(304, "W/\"0\"", etagSent()); + } + + @Test + public void IfNoneMatch_OtherTag_200() throws IOException { + Configure(null); + validateIfNoneMatch(200, "W/\"0\""); + } + + /** + * A field that is neither "*" nor a list of entity tags does not match, + * even when it holds the tag. Step 3. + */ + @Test + public void IfNoneMatch_TagThenNonTag_200() throws IOException { + Configure(null); + validateIfNoneMatch(200, etagSent() + ", other"); + } + + @Test + public void IfNoneMatch_TwoTagsWithNoComma_200() throws IOException { + Configure(null); + validateIfNoneMatch(200, "W/\"0\" " + etagSent()); + } + + /** + * A space is not allowed inside an entity tag's quotes. RFC 9110 section + * 8.8.3. + */ + @Test + public void IfNoneMatch_SpaceInsideQuotes_200() throws IOException { + Configure(null); + validateIfNoneMatch(200, "W/\"0 1\", " + etagSent()); + } + + /** + * "*" in a list with other values is not allowed. RFC 9110 section + * 13.1.2. + */ + @Test + public void IfNoneMatch_StarAndTag_200() throws IOException { + Configure(null); + validateIfNoneMatch(200, "*", etagSent()); + } + + /** + * The weak prefix is case sensitive. RFC 9110 section 8.8.3. + */ + @Test + public void IfNoneMatch_LowerCaseWeakPrefix_200() throws IOException { + Configure(null); + validateIfNoneMatch(200, "w" + etagSent().substring(1)); + } + + /** + * An opaque tag is quoted. RFC 9110 section 8.8.3. + */ + @Test + public void IfNoneMatch_Unquoted_200() throws IOException { + Configure(null); + String etag = etagSent(); + validateIfNoneMatch(200, etag.substring(3, etag.length() - 1)); + } + + /** + * The Content-Length header is the length of the content in bytes, in + * the character encoding the response is written with. + */ + @Test + public void Json_ContentLength_InBytes() throws IOException { + String json = "{\"city\":\"Zürich\"}"; + when(jsonData.getJson()).thenReturn(json); + when(response.getCharacterEncoding()).thenReturn("UTF-8"); + Configure(null); + + service.serveJson(request, response); + + verify(response).setContentLength(eq(json.length() + 1)); + } + + /** + * When the builder element is the one that failed there is no content, + * and the response is still sent rather than throwing. + */ + @Test + public void JavaScript_ProcessingErrors_NoContent() throws IOException { + recordProcessingError(); + when(flowData.getFromElement(jsElement)).thenReturn(null); + Configure(null); + + service.serveJavascript(request, response); + + ValidateFailedResponse("", JS_CONTENT_TYPE); + } + + /** + * Before processing, a matching If-None-Match gets a 304 carrying the + * headers of a good response. + */ + @Test + public void JavaScript_NotModified_Matching_304() throws IOException { + Configure(ConfigureElements( + Collections.singletonList(Arrays.asList("header.User-Agent")))); + String etag = etagSent(); + ifNoneMatch(etag); + + assertTrue( + service.serveJavascriptNotModified(flowData, request, response)); + + Validate304(etag); + } + + /** + * Before processing, a POST whose If-None-Match matches gets a 412. + */ + @Test + public void Json_NotModified_Post_412() throws IOException { + Configure(null); + String etag = etagSent(); + when(request.getMethod()).thenReturn("POST"); + ifNoneMatch(etag); + + assertTrue(service.serveJsonNotModified(flowData, request, response)); + + verify(response).setStatus(eq(412)); + verify(response, times(0)).setHeader(eq("ETag"), anyString()); + assertEquals("", responseString.toString()); + } + + /** + * An If-None-Match that does not match leaves the request to be processed, + * with nothing written. + */ + @Test + public void JavaScript_NotModified_OtherTag_NotAnswered() throws IOException { + Configure(null); + ifNoneMatch("W/\"other\""); + + assertFalse( + service.serveJavascriptNotModified(flowData, request, response)); + + verify(response, times(0)).setStatus(anyInt()); + verify(response, times(0)).setHeader(anyString(), anyString()); + } + + /** + * A Pipeline without the builder for the endpoint is left to be + * processed, which reports the configuration error. + */ + @Test + public void Json_NotModified_MissingBuilder_NotAnswered() { + Configure(null); + ifNoneMatch("*"); + when(pipeline.getElement(JsonBuilderElement.class)).thenReturn(null); + + assertFalse(service.serveJsonNotModified(flowData, request, response)); + + verify(response, times(0)).setStatus(anyInt()); + } + + /** + * A service whose Pipeline is not set yet answers nothing. + */ + @Test + public void NotModified_NoPipeline_NotAnswered() { + service = new ClientsidePropertyServiceCore.Default( + flowDataProvider, null); + ifNoneMatch("*"); + + assertFalse( + service.serveJavascriptNotModified(flowData, request, response)); + verify(response, times(0)).setStatus(anyInt()); + } + + /** + * The ETag the JavaScript endpoint sends for the request as it stands, + * from a response of its own. + */ + private String etagSent() throws IOException { + HttpServletResponse sent = mock(HttpServletResponse.class); + when(sent.getWriter()).thenReturn(new PrintWriter(new StringWriter())); + service.serveJavascript(request, sent); + ArgumentCaptor etag = ArgumentCaptor.forClass(String.class); + verify(sent).setHeader(eq("ETag"), etag.capture()); + return etag.getValue(); + } + + /** + * Give the request one If-None-Match field line for each value. getHeader + * gives the first line, as a servlet container does. + */ + private void ifNoneMatch(String... lines) { + when(request.getHeaders("If-None-Match")).thenAnswer( + invocation -> Collections.enumeration(Arrays.asList(lines))); + when(request.getHeader("If-None-Match")).thenReturn(lines[0]); + } + + private void validateIfNoneMatch(int expectedStatus, String... lines) + throws IOException { + ifNoneMatch(lines); + + service.serveJavascript(request, response); + + verify(response).setStatus(eq(expectedStatus)); + assertEquals( + expectedStatus == 304 ? "" : JS_CONTENT, + responseString.toString()); + } + + private void Validate304(String etag) { + verify(response).setStatus(eq(304)); + verify(response, times(0)).setStatus(eq(200)); + assertEquals("", responseString.toString()); + verify(response).setHeader( + eq("Cache-Control"), eq("private,max-age=1800")); + verify(response).setHeader(eq("ETag"), eq(etag)); + verify(response).setHeader(eq("Vary"), eq("User-Agent")); + verify(response).setHeader(eq("Access-Control-Allow-Origin"), eq("*")); + verify(response, times(0)).setContentLength(anyInt()); + verify(response, times(0)).setContentType(anyString()); + } + + /** + * A Pipeline with the Sequence Element, which adds a new session id to a + * request that has none, an element that accepts the User-Agent header, + * and the JSON builder. + */ + private static Pipeline webPipeline() throws Exception { + return new PipelineBuilder(LoggerFactory.getILoggerFactory()) + .addFlowElement(new SequenceElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new UserAgentElement()) + .addFlowElement(new JsonBuilderElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .setAutoCloseElements(true) + .build(); + } + + /** + * A GET request with a User-Agent header and no session id. + */ + private static HttpServletRequest webRequest(String userAgent) { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getMethod()).thenReturn("GET"); + when(request.getRequestURI()).thenReturn("/51dpipeline/json"); + when(request.getHeaderNames()).thenReturn( + Collections.enumeration(Arrays.asList("User-Agent", "X-Test"))); + when(request.getHeader("User-Agent")).thenReturn(userAgent); + when(request.getHeader("X-Test")).thenReturn("not accepted"); + when(request.getRemoteAddr()).thenReturn("1.2.3.4"); + when(request.getScheme()).thenReturn("https"); + return request; + } + + /** + * Add the request's Evidence to the FlowData as the web integration does, + * process it, and return the ETag the JSON endpoint sends. + */ + private static String servedETag( + Pipeline webPipeline, + FlowData data, + HttpServletRequest request) throws IOException { + new WebRequestEvidenceServiceCore.Default() + .addEvidenceFromRequest(data, request); + data.process(); + assertTrue("Processing recorded errors: " + data.getErrors(), + data.getErrors().isEmpty()); + + FlowDataProviderCore provider = mock(FlowDataProviderCore.class); + when(provider.getFlowData(request)).thenReturn(data); + HttpServletResponse served = mock(HttpServletResponse.class); + when(served.getWriter()).thenReturn(new PrintWriter(new StringWriter())); + new ClientsidePropertyServiceCore.Default(provider, webPipeline) + .serveJson(request, served); + ArgumentCaptor etag = ArgumentCaptor.forClass(String.class); + verify(served).setHeader(eq("ETag"), etag.capture()); + return etag.getValue(); + } + + /** + * An element that accepts the User-Agent header and adds nothing, in + * place of an engine. + */ + private static class UserAgentElement + extends FlowElementBase { + + private final EvidenceKeyFilter filter = new EvidenceKeyFilterWhitelist( + Collections.singletonList("header.user-agent"), + String.CASE_INSENSITIVE_ORDER); + + UserAgentElement() { + super(LoggerFactory.getLogger(UserAgentElement.class), null); + } + + @Override + protected void processInternal(FlowData data) { + } + + @Override + public String getElementDataKey() { + return "user-agent-test"; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return filter; + } + + @Override + public List getProperties() { + return new ArrayList<>(); + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + } + + private void recordProcessingError() { + when(flowData.getErrors()).thenReturn( + Collections.singletonList(mock(FlowError.class))); + } + + private void ValidateFailedResponse( + String expectedContent, + String contentType) { + assertEquals(expectedContent, responseString.toString()); + verify(response).setStatus(eq(200)); + verify(response).setContentType(eq(contentType)); + verify(response).setContentLength(eq(expectedContent.length())); + verify(response).setHeader(eq("Cache-Control"), eq("private, no-cache")); + verify(response, times(0)).setHeader(eq("ETag"), anyString()); + verify(response).setHeader(eq("Access-Control-Allow-Origin"), eq("*")); + } + private void JavaScript_VaryHeader( - List> elements, + List> elements, String expectedVary) throws IOException { Configure(ConfigureElements(elements)); @@ -220,16 +1062,16 @@ private void JavaScript_VaryHeader( JS_CONTENT.length(), JS_CONTENT_TYPE, 200, - String.valueOf(dataKey.hashCode()), + WEAK_ETAG, expectedVary); } private void ValidateResponse( String expectedContent, - Integer contentLength, + Integer contentLength, String contentType, int expectedStatusCode, - String expectedETag, + String expectedETagPattern, String expectedVary) { if (expectedContent == null) { expectedContent = ""; } @@ -269,8 +1111,8 @@ private void ValidateResponse( verify(response, times(0)).setHeader(eq("Vary"), anyString()); } - if (expectedETag != null) { - verify(response).setHeader(eq("ETag"), eq(expectedETag)); + if (expectedETagPattern != null) { + verify(response).setHeader(eq("ETag"), matches(expectedETagPattern)); } else { verify(response, times(0)).setHeader(eq("ETag"), anyString()); diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/FiftyOneJSServiceCoreTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/FiftyOneJSServiceCoreTests.java index c0f8ae388..cf9819fe4 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/FiftyOneJSServiceCoreTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/FiftyOneJSServiceCoreTests.java @@ -22,8 +22,10 @@ package fiftyone.pipeline.web; +import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.web.services.ClientsidePropertyServiceCore; import fiftyone.pipeline.web.services.FiftyOneJSServiceCore; +import fiftyone.pipeline.web.services.PipelineResultServiceCore; import org.junit.Before; import org.junit.Test; @@ -185,4 +187,95 @@ public void ClientSideDisabled_NothingWritten() throws IOException { verify(clientside, never()).serveJavascript(request, response); verify(clientside, never()).serveJson(request, response); } + + // The tests below stub and verify createFlowData on a mock, which + // creates nothing to close, so the FlowData rule does not apply. + + /** + * A request with no If-None-Match is left to be processed, and no + * FlowData is created for it. + */ + @Test + @SuppressWarnings("PMD.FlowDataInTryWithResources") + public void NotModified_NoIfNoneMatch_NothingCreated() throws IOException { + FiftyOneJSServiceCore service = + new FiftyOneJSServiceCore.Default(clientside, true); + PipelineResultServiceCore results = + mock(PipelineResultServiceCore.class); + requestPath("/51Degrees.core.js"); + + assertFalse(service.serveNotModified(request, response, results)); + + verify(results, never()).createFlowData(request); + } + + /** + * A request for another path is left to be processed, whatever its + * If-None-Match. + */ + @Test + @SuppressWarnings("PMD.FlowDataInTryWithResources") + public void NotModified_OtherPath_NothingCreated() throws IOException { + FiftyOneJSServiceCore service = + new FiftyOneJSServiceCore.Default(clientside, true); + PipelineResultServiceCore results = + mock(PipelineResultServiceCore.class); + when(request.getHeader("If-None-Match")).thenReturn("*"); + requestPath("/page"); + + assertFalse(service.serveNotModified(request, response, results)); + + verify(results, never()).createFlowData(request); + } + + /** + * With the client-side features off, nothing is answered early. + */ + @Test + @SuppressWarnings("PMD.FlowDataInTryWithResources") + public void NotModified_ClientSideDisabled_NothingCreated() + throws IOException { + FiftyOneJSServiceCore service = + new FiftyOneJSServiceCore.Default(clientside, false); + PipelineResultServiceCore results = + mock(PipelineResultServiceCore.class); + when(request.getHeader("If-None-Match")).thenReturn("*"); + requestPath("/51Degrees.core.js"); + + assertFalse(service.serveNotModified(request, response, results)); + + verify(results, never()).createFlowData(request); + } + + /** + * A revalidation of the JavaScript or the JSON goes to its own endpoint + * with a FlowData created for it, which is closed afterwards. + */ + @Test + @SuppressWarnings("PMD.FlowDataInTryWithResources") + public void NotModified_JavaScriptAndJson_AskTheirEndpoint() + throws Exception { + FiftyOneJSServiceCore service = + new FiftyOneJSServiceCore.Default(clientside, true); + PipelineResultServiceCore results = + mock(PipelineResultServiceCore.class); + FlowData flowData = mock(FlowData.class); + when(results.createFlowData(request)).thenReturn(flowData); + when(request.getHeader("If-None-Match")).thenReturn("*"); + when(clientside.serveJavascriptNotModified(flowData, request, response)) + .thenReturn(true); + when(clientside.serveJsonNotModified(flowData, request, response)) + .thenReturn(true); + + requestPath("/51Degrees.core.js"); + assertTrue(service.serveNotModified(request, response, results)); + requestPath("/51dpipeline/json"); + assertTrue(service.serveNotModified(request, response, results)); + + verify(clientside, times(1)) + .serveJavascriptNotModified(flowData, request, response); + verify(clientside, times(1)) + .serveJsonNotModified(flowData, request, response); + verify(flowData, times(2)).close(); + } } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/FlowDataThreadsDocumentationTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/FlowDataThreadsDocumentationTests.java new file mode 100644 index 000000000..384f165fb --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/FlowDataThreadsDocumentationTests.java @@ -0,0 +1,236 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web; + +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Checks that the web integration documentation and every web integration + * example say that a Flow Data cannot be used by two threads at once, and + * show that work handed to another thread gets values, not the Flow Data. + * Reads source and documentation only. + */ +public class FlowDataThreadsDocumentationTests { + + private static final String STATEMENT = + "a flow data cannot be used by two threads at once"; + + private static final String HAND_OFF = "not the flow data"; + + private static final Pattern LINE_LEADERS = + Pattern.compile("(?m)^\\s*(\\*|//|#|>)+"); + + private static final Pattern WEB_IMPORT = Pattern.compile( + "import\\s+(javax\\.servlet|jakarta\\.servlet|" + + "org\\.springframework\\.web)"); + + private static final List DOCUMENTS = Arrays.asList( + "pipeline/README.md", + "examples/device-detection/README.md", + "examples/ip-intelligence/README.md", + "location/README.md", + "pipeline/pipeline.web.packages/pipeline.web/src/main/java/" + + "fiftyone/pipeline/web/PipelineFilter.java", + "pipeline/pipeline.web.packages/pipeline.web/src/main/java/" + + "fiftyone/pipeline/web/services/FlowDataProviderCore.java", + "pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/" + + "fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java", + "pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/" + + "fiftyone/pipeline/web/mvc/components/FlowDataProvider.java"); + + private static final List KNOWN_EXAMPLES = Arrays.asList( + "examples/device-detection/web/getting-started.cloud/src/main/java/" + + "fiftyone/devicedetection/examples/web/GettingStartedWebCloud.java", + "examples/device-detection/web/getting-started.onprem/src/main/" + + "java/fiftyone/devicedetection/examples/web/" + + "GettingStartedWebOnPrem.java", + "examples/ip-intelligence/web/getting-started.api/src/main/java/" + + "fiftyone/ipintelligence/examples/web/GettingStartedApi.java", + "examples/ip-intelligence/web/getting-started.cloud/src/main/java/" + + "fiftyone/ipintelligence/examples/web/GettingStartedWebCloud.java", + "examples/ip-intelligence/web/getting-started.cloud.mixed/src/main/" + + "java/fiftyone/ipintelligence/examples/web/" + + "GettingStartedWebCloudMixed.java", + "examples/ip-intelligence/web/getting-started.mixed/src/main/java/" + + "fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java", + "examples/ip-intelligence/web/getting-started.onprem/src/main/java/" + + "fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java", + "location/geo-location.web.examples/geo-location.web.examples.mvc/" + + "src/main/java/fiftyone/geolocation/web/examples/mvc/" + + "configuration/ExampleMvcConfig.java", + "location/geo-location.web.examples/geo-location.web.examples.mvc/" + + "src/main/java/fiftyone/geolocation/web/examples/mvc/" + + "controller/ExampleController.java", + "location/geo-location.web.examples/" + + "geo-location.web.examples.servlet/src/main/java/fiftyone/" + + "geolocation/web/examples/servlet/Example.java", + "pipeline/pipeline.developer-examples/" + + "pipeline.developer-examples.clientside-element-mvc/src/main/" + + "java/controller/ExampleController.java"); + + @Test + public void theWebIntegrationDocumentationSaysIt() throws IOException { + List missing = new ArrayList<>(); + for (String document : DOCUMENTS) { + String text = normalised(root().resolve(document)); + if (text.contains(STATEMENT) == false || + text.contains(HAND_OFF) == false) { + missing.add(document); + } + } + assertEquals( + "These do not say that a Flow Data cannot be used by two " + + "threads at once, with values, not the Flow Data, handed " + + "to other threads", + new ArrayList(), + missing); + } + + @Test + public void thePipelineReadmeShowsAHandOff() throws IOException { + String readme = new String( + Files.readAllBytes(root().resolve("pipeline/README.md")), + StandardCharsets.UTF_8); + int section = readme.indexOf("Flow Data and threads"); + assertTrue( + "pipeline/README.md has no Flow Data and threads section", + section >= 0); + String rest = readme.substring(section); + assertTrue( + "The Flow Data and threads section shows no work handed to " + + "an executor", + rest.contains(".submit(")); + } + + @Test + public void everyWebExampleSaysIt() throws IOException { + Path root = root(); + List found = webExamples(root); + for (String known : KNOWN_EXAMPLES) { + assertTrue( + "The web example " + known + " was not found", + found.contains(known)); + } + List missing = new ArrayList<>(); + for (String example : found) { + String text = normalised(root.resolve(example)); + if (text.contains(STATEMENT) == false || + text.contains(HAND_OFF) == false) { + missing.add(example); + } + } + assertEquals( + "These web examples do not say that a Flow Data cannot be " + + "used by two threads at once, with values, not the Flow " + + "Data, handed to other threads", + new ArrayList(), + missing); + } + + /** + * Every main Java source outside the web packages that uses a servlet + * or Spring web API and gets or creates a Flow Data. + */ + private static List webExamples(Path root) throws IOException { + try (Stream paths = Files.walk(root)) { + List sources = paths + .filter(p -> p.toString().endsWith(".java")) + .filter(p -> isExampleSource(root.relativize(p))) + .collect(Collectors.toList()); + List examples = new ArrayList<>(); + for (Path source : sources) { + String text = new String( + Files.readAllBytes(source), + StandardCharsets.UTF_8); + if (WEB_IMPORT.matcher(text).find() && + (text.contains("getFlowData(") || + text.contains("createFlowData("))) { + examples.add(relative(root, source)); + } + } + return examples; + } + } + + private static boolean isExampleSource(Path relative) { + String path = relative.toString().replace(File.separatorChar, '/'); + return path.contains("/src/main/java/") && + path.startsWith("cxx/") == false && + path.startsWith("pipeline/pipeline.web.packages/") == false && + path.contains("/target/") == false && + path.contains("/node_modules/") == false; + } + + private static String relative(Path root, Path path) { + return root.relativize(path).toString() + .replace(File.separatorChar, '/'); + } + + /** + * The text with comment and Markdown line leaders removed, white space + * collapsed and in lower case, so a sentence wrapped over lines matches. + */ + private static String normalised(Path path) throws IOException { + String text = new String( + Files.readAllBytes(path), + StandardCharsets.UTF_8); + return LINE_LEADERS.matcher(text).replaceAll(" ") + .replaceAll("\\s+", " ") + .toLowerCase(); + } + + /** + * The repository root, being the nearest folder above the working + * folder that holds the pipeline, examples and location folders. + */ + private static Path root() { + File folder = new File(System.getProperty("user.dir")) + .getAbsoluteFile(); + while (folder != null) { + if (new File(folder, "pipeline").isDirectory() && + new File(folder, "examples").isDirectory() && + new File(folder, "location").isDirectory()) { + return folder.toPath(); + } + folder = folder.getParentFile(); + } + assertNotNull("The repository root was not found", folder); + return null; + } +} diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterErrorTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterErrorTests.java new file mode 100644 index 000000000..19bbb29cd --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterErrorTests.java @@ -0,0 +1,228 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.web.services.FiftyOneJSServiceCore; +import fiftyone.pipeline.web.services.FlowDataProviderCore; +import fiftyone.pipeline.web.services.PipelineResultServiceCore; +import fiftyone.pipeline.web.services.UACHServiceCore; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.stubbing.Answer; + +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; + +import static fiftyone.pipeline.web.Constants.HTTPCONTEXT_FLOWDATA; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * An error in the web integration does not stop the customer's request + * when the Pipeline suppresses process exceptions: it is logged and the + * request goes on down the filter chain. When the Pipeline does not + * suppress them, the error is thrown as before. See + * exception-handling.md line 73: + * "The implementor MUST ensure that exceptions/errors that happen in these + * areas are handled correctly in order to avoid the API taking down the + * customer's production website." and line 42, which gives + * IsSuppressProcessExceptions to wrappers "to make a decision on whether the + * exceptions should be suppressed and logged during processing." + */ +public class PipelineFilterErrorTests { + + private PipelineFilter filter; + private HttpServletRequest request; + private HttpServletResponse response; + private FilterChain chain; + private Object storedFlowData; + + @Before + public void init() throws Exception { + FilterConfig config = mock(FilterConfig.class); + when(config.getInitParameter(anyString())).thenAnswer( + (Answer) invocation -> + "config-file".equals(invocation.getArgument(0)) ? + "filename" : null); + ServletContext context = mock(ServletContext.class); + File file = new File(getClass().getClassLoader() + .getResource("config.xml").getFile()); + when(context.getRealPath(anyString())) + .thenReturn(file.getAbsolutePath()); + when(config.getServletContext()).thenReturn(context); + + request = mock(HttpServletRequest.class); + response = mock(HttpServletResponse.class); + chain = mock(FilterChain.class); + when(request.getScheme()).thenReturn(""); + when(request.getLocalAddr()).thenReturn(""); + when(request.getRequestURL()).thenReturn(new StringBuffer("")); + when(request.getRequestURI()).thenReturn(""); + when(request.getAttribute(eq(HTTPCONTEXT_FLOWDATA))) + .thenAnswer(invocation -> storedFlowData); + org.mockito.Mockito.doAnswer(invocation -> { + storedFlowData = invocation.getArgument(1); + return null; + }).when(request).setAttribute(eq(HTTPCONTEXT_FLOWDATA), any()); + + filter = new PipelineFilter(); + filter.init(config); + } + + @After + public void cleanUp() { + if (filter != null) { + filter.destroy(); + } + } + + private void set(String name, Object value) throws Exception { + Field field = PipelineFilter.class.getDeclaredField(name); + field.setAccessible(true); + field.set(filter, value); + } + + private Object get(String name) throws Exception { + Field field = PipelineFilter.class.getDeclaredField(name); + field.setAccessible(true); + return field.get(filter); + } + + /** + * Give the filter a Pipeline that reports the setting, keeping the + * real one to close. + */ + private void suppress(boolean value) throws Exception { + Pipeline real = (Pipeline) get("pipeline"); + Pipeline reporting = mock(Pipeline.class, + org.mockito.AdditionalAnswers.delegatesTo(real)); + when(reporting.isSuppressProcessExceptions()).thenReturn(value); + set("pipeline", reporting); + } + + private void failProcessing(RuntimeException failure) throws Exception { + PipelineResultServiceCore failing = + mock(PipelineResultServiceCore.class); + doThrow(failure).when(failing).process(any(HttpServletRequest.class)); + set("resultService", failing); + } + + @Test + public void processingFails_Suppressed_RequestGoesOn() throws Exception { + suppress(true); + failProcessing(new IllegalStateException("processing failed")); + + filter.doFilter(request, response, chain); + + verify(chain, times(1)).doFilter( + any(ServletRequest.class), any(ServletResponse.class)); + } + + @Test + public void headersFail_Suppressed_RequestGoesOn() throws Exception { + suppress(true); + UACHServiceCore failing = mock(UACHServiceCore.class); + doThrow(new IllegalStateException("headers failed")).when(failing) + .setResponseHeaders(any(FlowData.class), + any(HttpServletResponse.class)); + set("uachServiceCore", failing); + + filter.doFilter(request, response, chain); + + verify(chain, times(1)).doFilter( + any(ServletRequest.class), any(ServletResponse.class)); + } + + @Test + public void processingFails_NotSuppressed_IsThrown() throws Exception { + suppress(false); + IllegalStateException failure = + new IllegalStateException("processing failed"); + failProcessing(failure); + + try { + filter.doFilter(request, response, chain); + fail("The failure was not thrown."); + } catch (IllegalStateException e) { + assertSame(failure, e); + } catch (ServletException e) { + fail("The failure was wrapped: " + e); + } + } + + /** + * Serving the 51Degrees JavaScript fails with an IOException, for + * example when the client has gone away. The failure is thrown, and the + * Flow Data is still closed, so its Element Data and native results are + * released. + */ + @Test + public void servingJavaScriptFails_IOException_FlowDataClosed() + throws Exception { + suppress(true); + set("resultService", mock(PipelineResultServiceCore.class)); + set("uachServiceCore", mock(UACHServiceCore.class)); + FlowData flowData = mock(FlowData.class); + FlowDataProviderCore provider = mock(FlowDataProviderCore.class); + when(provider.getFlowData(any(HttpServletRequest.class))) + .thenReturn(flowData); + set("flowDataProviderCore", provider); + FiftyOneJSServiceCore js = mock(FiftyOneJSServiceCore.class); + IOException failure = new IOException("client went away"); + when(js.serveJS(any(HttpServletRequest.class), + any(HttpServletResponse.class))).thenThrow(failure); + set("jsService", js); + + try { + filter.doFilter(request, response, chain); + fail("The failure was not thrown."); + } catch (IOException e) { + assertSame(failure, e); + } + + verify(flowData, times(1)).close(); + verify(chain, never()).doFilter( + any(ServletRequest.class), any(ServletResponse.class)); + } +} diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java index d0e60d9e4..7369541ac 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java @@ -51,7 +51,11 @@ import javax.servlet.http.HttpServletResponse; import java.io.File; import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -148,6 +152,57 @@ public void PipelineFilter_Initialise() throws ServletException { filter.init(config); } + /** + * The client-side max-age is the default when the init parameter is + * not set. + * @throws ServletException + */ + @Test + public void CacheMaxAge_NotSet_IsTheDefault() throws ServletException { + assertEquals( + DEFAULT_CLIENTSIDE_CACHE_MAX_AGE_SECONDS, + PipelineFilter.getCacheMaxAgeSeconds(config)); + } + + @Test + public void CacheMaxAge_Set() throws ServletException { + FilterConfig withMaxAge = mock(FilterConfig.class); + when(withMaxAge.getInitParameter(INIT_PARAM_CLIENTSIDE_CACHE_MAX_AGE)) + .thenReturn(" 60 "); + + assertEquals(60, PipelineFilter.getCacheMaxAgeSeconds(withMaxAge)); + } + + /** + * A max-age that is not a whole number of seconds, 0 or more, fails + * the filter's start rather than being ignored. + */ + @Test + public void CacheMaxAge_Invalid_Refused() { + for (String value : new String[] { "-1", "1.5", "ten" }) { + FilterConfig invalid = mock(FilterConfig.class); + when(invalid.getInitParameter(INIT_PARAM_CLIENTSIDE_CACHE_MAX_AGE)) + .thenReturn(value); + try { + PipelineFilter.getCacheMaxAgeSeconds(invalid); + fail("'" + value + "' was accepted"); + } catch (ServletException e) { + assertTrue(e.getMessage(), e.getMessage().contains( + INIT_PARAM_CLIENTSIDE_CACHE_MAX_AGE)); + } + } + } + + @Test(expected = ServletException.class) + public void PipelineFilter_Initialise_InvalidCacheMaxAge() + throws ServletException { + when(config.getInitParameter(INIT_PARAM_CLIENTSIDE_CACHE_MAX_AGE)) + .thenReturn("-1"); + filter = new PipelineFilter(); + + filter.init(config); + } + /** * Check that the filter can process a request, and that it correctly adds * a processed FlowData to the request. @@ -382,4 +437,110 @@ public void StartupHelpers_NoJsonPath_IsTheDefault() throws Exception { } } } + + /** + * A request for the JavaScript whose If-None-Match matches the ETag of + * the Evidence it supplies is answered with a 304 carrying the headers of + * a good response, without processing and without the rest of the chain. + */ + @Test + public void PipelineFilter_MatchingIfNoneMatch_304_NotProcessed() + throws Exception { + filter = new PipelineFilter(); + filter.init(config); + HttpServletRequest revalidation = + clientsideRequest("GET", "/" + CORE_JS_NAME, "*"); + HttpServletResponse answer = clientsideResponse(); + + filter.doFilter(revalidation, answer, chain); + + verify(revalidation, never()).setAttribute( + eq(HTTPCONTEXT_FLOWDATA), any()); + verify(answer).setStatus(304); + verify(answer).setHeader( + eq("Cache-Control"), eq("private,max-age=1800")); + verify(answer).setHeader(eq("ETag"), matches("W/\"[0-9a-f]{64}\"")); + verify(chain, never()).doFilter( + any(ServletRequest.class), any(ServletResponse.class)); + } + + /** + * A POST for the JSON whose If-None-Match matches gets a 412, without + * processing. + */ + @Test + public void PipelineFilter_MatchingIfNoneMatch_Post_412_NotProcessed() + throws Exception { + filter = new PipelineFilter(); + filter.init(config); + HttpServletRequest revalidation = + clientsideRequest("POST", DEFAULT_JSON_PATH, "*"); + HttpServletResponse answer = clientsideResponse(); + + filter.doFilter(revalidation, answer, chain); + + verify(revalidation, never()).setAttribute( + eq(HTTPCONTEXT_FLOWDATA), any()); + verify(answer).setStatus(412); + verify(chain, never()).doFilter( + any(ServletRequest.class), any(ServletResponse.class)); + } + + /** + * A request for the JavaScript whose If-None-Match does not match is + * processed and gets the JavaScript. + */ + @Test + public void PipelineFilter_OtherIfNoneMatch_Processed() throws Exception { + filter = new PipelineFilter(); + filter.init(config); + HttpServletRequest request = + clientsideRequest("GET", "/" + CORE_JS_NAME, "W/\"other\""); + HttpServletResponse answer = clientsideResponse(); + + filter.doFilter(request, answer, chain); + + verify(request, times(1)).setAttribute( + eq(HTTPCONTEXT_FLOWDATA), any(FlowData.class)); + verify(answer).setStatus(200); + verify(answer, never()).setStatus(304); + } + + /** + * A request for the given path with one If-None-Match field line, whose + * attributes are kept as a servlet container keeps them. + */ + private static HttpServletRequest clientsideRequest( + String method, + String path, + String ifNoneMatch) { + Map attributes = new HashMap<>(); + HttpServletRequest clientside = mock(HttpServletRequest.class); + when(clientside.getMethod()).thenReturn(method); + when(clientside.getRequestURI()).thenReturn(path); + when(clientside.getContextPath()).thenReturn(""); + when(clientside.getScheme()).thenReturn("https"); + when(clientside.getLocalAddr()).thenReturn(""); + when(clientside.getRequestURL()) + .thenReturn(new StringBuffer("https://example.com" + path)); + when(clientside.getHeaderNames()).thenAnswer(invocation -> + Collections.enumeration(Arrays.asList("Host", "If-None-Match"))); + when(clientside.getHeader("Host")).thenReturn("example.com"); + when(clientside.getHeader("If-None-Match")).thenReturn(ifNoneMatch); + when(clientside.getHeaders("If-None-Match")).thenAnswer(invocation -> + Collections.enumeration(Collections.singletonList(ifNoneMatch))); + doAnswer(invocation -> attributes.put( + invocation.getArgument(0), invocation.getArgument(1))) + .when(clientside).setAttribute(anyString(), any()); + when(clientside.getAttribute(anyString())).thenAnswer(invocation -> + attributes.get((String) invocation.getArgument(0))); + return clientside; + } + + private static HttpServletResponse clientsideResponse() throws IOException { + HttpServletResponse answer = mock(HttpServletResponse.class); + when(answer.getWriter()) + .thenReturn(new PrintWriter(new StringWriter())); + return answer; + } } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/RequestLineEvidenceTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/RequestLineEvidenceTests.java new file mode 100644 index 000000000..dec55d834 --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/RequestLineEvidenceTests.java @@ -0,0 +1,110 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web; + +import fiftyone.pipeline.core.data.Evidence; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.web.services.WebRequestEvidenceServiceCore; +import org.junit.Test; + +import javax.servlet.http.HttpServletRequest; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The request line Evidence of the web integration specification's + * Populating Evidence section. An integration that supplies these keys + * supplies all three on every request, with server.request-query empty + * when the request has no query string, and each exactly as it appeared on + * the request line. + */ +public class RequestLineEvidenceTests { + + private final WebRequestEvidenceServiceCore evidenceService = + new WebRequestEvidenceServiceCore.Default(); + + /** + * Flow Data whose Pipeline has an element that accepts the three request + * line keys. + */ + private static FlowData flowData() { + FlowData flowData = mock(FlowData.class); + Evidence evidence = mock(Evidence.class); + when(evidence.asKeyMap()).thenReturn(new HashMap()); + when(flowData.getEvidence()).thenReturn(evidence); + when(flowData.getEvidenceKeyFilter()).thenReturn( + new EvidenceKeyFilterWhitelist(Arrays.asList( + "server.request-method", + "server.request-path", + "server.request-query"))); + return flowData; + } + + private static HttpServletRequest request(String path, String query) { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getMethod()).thenReturn("GET"); + when(request.getRequestURI()).thenReturn(path); + when(request.getQueryString()).thenReturn(query); + when(request.getHeaderNames()) + .thenReturn(Collections.emptyEnumeration()); + when(request.getParameterNames()) + .thenReturn(Collections.emptyEnumeration()); + return request; + } + + /** + * The query string is supplied whole and undecoded, beside the method + * and the path. + */ + @Test + public void RequestQuery_SuppliedWholeAndUndecoded() { + FlowData flowData = flowData(); + + evidenceService.addEvidenceFromRequest( + flowData, request("/a", "x=%2F&y=1")); + + verify(flowData).addEvidence("server.request-method", "GET"); + verify(flowData).addEvidence("server.request-path", "/a"); + verify(flowData).addEvidence("server.request-query", "x=%2F&y=1"); + } + + /** + * A request with no query string still gets server.request-query, empty, + * so an absent key only ever means the request line is not supplied. + */ + @Test + public void RequestQuery_NoQueryString_SuppliedEmpty() { + FlowData flowData = flowData(); + + evidenceService.addEvidenceFromRequest(flowData, request("/a", null)); + + verify(flowData).addEvidence("server.request-path", "/a"); + verify(flowData).addEvidence("server.request-query", ""); + } +} diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java index b06d5e8b2..3107ffedf 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java @@ -37,21 +37,39 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import com.sun.net.httpserver.HttpServer; import org.mockito.ArgumentCaptor; import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.io.PrintWriter; import java.io.StringWriter; +import java.net.InetAddress; +import java.net.InetSocketAddress; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.zip.GZIPInputStream; + +import javax.xml.parsers.DocumentBuilderFactory; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; @@ -75,6 +93,12 @@ public class UsageSharingEvidenceTests { private static final String SESSION_ID = "1f2e3d4c-5b6a-4987-8a1b-2c3d4e5f6a7b"; + /** + * How long to wait for the usage sharing message to arrive. It is sent + * at once, so this is only an upper bound for a slow runner. + */ + private static final int MESSAGE_WAIT_SECONDS = 60; + private Pipeline pipeline; private final WebRequestEvidenceServiceCore evidenceService = @@ -109,6 +133,9 @@ private static HttpServletRequest request(Map headers) { for (Map.Entry header : headers.entrySet()) { when(request.getHeader(header.getKey())) .thenReturn(header.getValue()); + when(request.getHeaders(header.getKey())).thenAnswer( + invocation -> Collections.enumeration( + Collections.singletonList(header.getValue()))); } when(request.getCookies()).thenReturn(new Cookie[]{ new Cookie("51D_Profile", "123456")}); @@ -272,17 +299,168 @@ public void Revalidation_SameETag_Gets304() throws Exception { when(provider.getFlowData(revalidation)) .thenReturn(revalidationData); - assertEquals("The revalidation's ETag differs from the one it sends.", - etag.getValue(), - Integer.toString(revalidationData.generateKey( - pipeline.getEvidenceKeyFilter()).hashCode())); - HttpServletResponse revalidationResponse = response(); service.serveJson(revalidation, revalidationResponse); verify(revalidationResponse).setStatus(304); verify(revalidationResponse, never()).setStatus(200); verify(revalidationResponse, never()).setContentType(anyString()); + // The 304 carries the ETag a 200 would, which is the one sent. + verify(revalidationResponse).setHeader("ETag", etag.getValue()); + } + } + + /** + * A header no element reads, sent through the web integration, reaches + * the usage sharing message. The message is sent over HTTP to a local + * endpoint, as it would be to 51Degrees, and read back from there. + */ + @Test + public void UnreadHeader_ReachesTheUsageSharingMessage() throws Exception { + BlockingQueue received = new LinkedBlockingQueue<>(); + HttpServer server = messageServer(received); + try (Pipeline sharing = sharingPipeline(server)) { + assertFalse("A header no element uses is accepted.", + sharing.getEvidenceKeyFilter().include("header.x-test")); + try (FlowData flowData = sharing.createFlowData()) { + evidenceService.addEvidenceFromRequest( + flowData, request(headers())); + flowData.process(); + } + + Document message = receivedMessage(received); + assertEquals("The header no element reads was not shared.", + "shared", sharedValue(message, "header", "X-Test")); + assertEquals("The User-Agent was not shared.", + USER_AGENT, sharedValue(message, "header", "User-Agent")); + assertNull("The raw Cookie header was shared.", + sharedValue(message, "header", "Cookie")); + } finally { + server.stop(0); + } + } + + /** + * Values an application adds to the Evidence beside the web request + * reach the usage sharing message with the request's own values. They + * are the JA4 fingerprint, the HTTP/2 fingerprint and the two billing + * keys, which the specification says are shared whatever the + * configuration. + */ + @Test + public void AddedEvidence_ReachesTheUsageSharingMessage() throws Exception { + Map added = new LinkedHashMap<>(); + added.put("server.ja4", "t13d1516h2_8daaf6152771_02713d6af862"); + added.put("server.h2", + "1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p"); + added.put("fiftyone.resource-key", "test-resource-key"); + added.put("fiftyone.license-key", "test-license-key"); + BlockingQueue received = new LinkedBlockingQueue<>(); + HttpServer server = messageServer(received); + try (Pipeline sharing = sharingPipeline(server)) { + try (FlowData flowData = sharing.createFlowData()) { + evidenceService.addEvidenceFromRequest( + flowData, request(headers())); + for (Map.Entry value : added.entrySet()) { + flowData.addEvidence(value.getKey(), value.getValue()); + } + flowData.process(); + } + + Document message = receivedMessage(received); + for (Map.Entry value : added.entrySet()) { + String[] parts = value.getKey().split("\\.", 2); + assertEquals(value.getKey() + " was not shared.", + value.getValue(), + sharedValue(message, parts[0], parts[1])); + } + assertEquals("The request's own header was not shared.", + "shared", sharedValue(message, "header", "X-Test")); + } finally { + server.stop(0); + } + } + + /** + * Starts a local HTTP endpoint that adds the body of each request it + * receives to the queue, as 51Degrees receives a usage sharing message. + */ + private static HttpServer messageServer(BlockingQueue received) + throws IOException { + HttpServer server = HttpServer.create( + new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + server.createContext("/", exchange -> { + try (InputStream body = exchange.getRequestBody()) { + received.add(readAll(body)); + } + exchange.sendResponseHeaders(200, -1); + exchange.close(); + }); + server.start(); + return server; + } + + /** + * A Pipeline whose Usage Sharing Element sends each request to the local + * endpoint at once, with no repeat interval. + */ + private static Pipeline sharingPipeline(HttpServer server) + throws Exception { + return new PipelineBuilder(LoggerFactory.getILoggerFactory()) + .addFlowElement(new SequenceElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new ShareUsageBuilder( + LoggerFactory.getILoggerFactory()) + .setSharePercentage(1) + .setMinimumEntriesPerMessage(1) + .setRepeatEvidenceIntervalMinutes(0) + .setShareUsageUrl("http://" + + server.getAddress().getHostString() + ":" + + server.getAddress().getPort() + "/new.ashx") + .build()) + .setAutoCloseElements(true) + .build(); + } + + /** + * The first usage sharing message the local endpoint receives, unzipped + * and parsed. + */ + private static Document receivedMessage(BlockingQueue received) + throws Exception { + byte[] sent = received.poll(MESSAGE_WAIT_SECONDS, TimeUnit.SECONDS); + assertNotNull("No usage sharing message was sent.", sent); + return DocumentBuilderFactory.newInstance() + .newDocumentBuilder() + .parse(new GZIPInputStream(new ByteArrayInputStream(sent))); + } + + /** + * The value the message gives for the named value of a category, such + * as header, or null when it gives none. Names are compared without + * regard to case. + */ + private static String sharedValue( + Document message, + String category, + String name) { + NodeList values = message.getElementsByTagName(category); + for (int i = 0; i < values.getLength(); i++) { + Element value = (Element) values.item(i); + if (name.equalsIgnoreCase(value.getAttribute("Name"))) { + return value.getTextContent(); + } + } + return null; + } + + private static byte[] readAll(InputStream input) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = input.read(buffer)) != -1) { + output.write(buffer, 0, read); } + return output.toByteArray(); } private static HttpServletResponse response() throws Exception { diff --git a/pipeline/pom.xml b/pipeline/pom.xml index abcd6d8f6..cd05ac720 100644 --- a/pipeline/pom.xml +++ b/pipeline/pom.xml @@ -54,6 +54,7 @@ pipeline.cloudrequestengine pipeline.engines.fiftyone pipeline.translation + pipeline.derivedproperty pipeline.javascriptbuilder pipeline.jsonbuilder pipeline.web.packages @@ -74,12 +75,14 @@ 20231013 0.11.7 1.10.0 + 2.7 3.0.1 3.0.2 3.0.2 0.9.14 + 1.0 22.0.0.2 22.0.0.2 @@ -95,7 +98,7 @@ 2.6 3.7 0.7.0 - 3.26.0 + 3.28.0 true false @@ -115,6 +118,11 @@ json ${org.json.version} + + org.snakeyaml + snakeyaml-engine + ${snakeyaml-engine.version} + @@ -406,9 +414,11 @@ - - https://github.com/51Degrees/pipeline-java - scm:git:git://github.com/51Degrees/pipeline-java.git - scm:git:info@51degrees.com:51Degrees/pipeline-java.git + + https://github.com/51Degrees/51degrees-java + scm:git:https://github.com/51Degrees/51degrees-java.git + scm:git:https://github.com/51Degrees/51degrees-java.git diff --git a/pom.xml b/pom.xml index 567875602..f638fd9df 100644 --- a/pom.xml +++ b/pom.xml @@ -9,13 +9,13 @@ nothing here reaches Maven Central and the published poms are the same shape as before the packages shared a repository. - Every module carries the same version. CI sets it on all of them - before a release with versions:set and processAllModules. The + Every module carries the same development version, and the libraries reference each other through ${project.version}, so a - change in pipeline-java is seen by device-detection-java in the same - build. The examples reference the packages published to Maven Central - unless the dev profile is active, which points them at the libraries - here instead. --> + change to the Pipeline API is seen by Device Detection in the same + build. A release gives each published module its own version, which + ci/build-package.ps1 writes into the poms. The examples reference the + packages published to Maven Central unless the dev profile is + active, which points them at the libraries here instead. --> com.51degrees 51degrees-java 4.5.127-SNAPSHOT