diff --git a/.github/workflows/ci-script-tests.yml b/.github/workflows/ci-script-tests.yml index 8f68b5e0..3a74d841 100644 --- a/.github/workflows/ci-script-tests.yml +++ b/.github/workflows/ci-script-tests.yml @@ -1,13 +1,16 @@ name: CI script tests # Runs the Pester tests of the scripts in ci/, which choose what a pull request -# builds and what a publish releases. common-ci's main is checked out beside +# builds and what a publish releases, and the test of how +# free-threaded-python.yml fetches the free data files. common-ci's main is +# checked out beside # this repository, where ci/dispatch.Tests.ps1 looks for it. on: pull_request: paths: - 'ci/**' - '.github/workflows/ci-script-tests.yml' + - '.github/workflows/free-threaded-python.yml' push: branches: - main @@ -15,6 +18,7 @@ on: paths: - 'ci/**' - '.github/workflows/ci-script-tests.yml' + - '.github/workflows/free-threaded-python.yml' workflow_dispatch: permissions: diff --git a/.github/workflows/free-threaded-python.yml b/.github/workflows/free-threaded-python.yml new file mode 100644 index 00000000..d1dc2c87 --- /dev/null +++ b/.github/workflows/free-threaded-python.yml @@ -0,0 +1,211 @@ +name: Free-threaded Python + +# Builds the Device Detection and IP Intelligence on-premise packages on the +# free-threaded builds of Python and runs their tests there, with the free +# Lite data files. The first job downloads those once for each commit of the +# data submodule, which links them, and puts them in the cache. The test jobs +# restore them from there and never download them. +# ci/free-threaded-python.Tests.ps1 checks this. The native modules do not declare that they run without +# the GIL, so Python turns the GIL back on when it loads them. +# tests/test_free_threaded.py in each package checks that it does, and is +# skipped on every other build of Python. +on: + pull_request: + paths: + - 'device-detection/fiftyone_devicedetection_onpremise/**' + - 'ip-intelligence/fiftyone_ipintelligence_onpremise/**' + - 'pipeline/fiftyone_pipeline_core/**' + - 'pipeline/fiftyone_pipeline_engines/**' + - 'pipeline/fiftyone_pipeline_engines_fiftyone/**' + - 'cxx' + - '.github/workflows/free-threaded-python.yml' + push: + branches: + - main + - rc + paths: + - 'device-detection/fiftyone_devicedetection_onpremise/**' + - 'ip-intelligence/fiftyone_ipintelligence_onpremise/**' + - 'pipeline/fiftyone_pipeline_core/**' + - 'pipeline/fiftyone_pipeline_engines/**' + - 'pipeline/fiftyone_pipeline_engines_fiftyone/**' + - 'cxx' + - '.github/workflows/free-threaded-python.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + free-data-files: + name: Fetch the free Lite data files once + runs-on: ubuntu-latest + outputs: + key: ${{ steps.key.outputs.key }} + defaults: + run: + shell: pwsh + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + submodules: recursive + + - name: Name the cache after the data submodule + id: key + run: | + # The data submodule's .link files say which files are fetched and + # from where, so a new commit of it needs the files again. Change + # v1 when the list of files below changes. + $data = git -C cxx rev-parse HEAD:data + if ($LASTEXITCODE) { exit $LASTEXITCODE } + "key=free-lite-data-v1-$data" >> $env:GITHUB_OUTPUT + + - name: Look for the files in the cache + id: cache + uses: actions/cache@v4 + with: + path: | + cxx/data/**/51Degrees-LiteV4.1.hash + cxx/data/**/20000 Evidence Records.yml + cxx/data/**/20000 User Agents.csv + cxx/data/**/51Degrees-LiteV41.ipi + cxx/data/**/51Degrees-IPIV4AsnIpiV41.ipi + key: ${{ steps.key.outputs.key }} + # The Windows job restores what this Linux job saves. + enableCrossOsArchive: true + lookup-only: true + + # Runs only when the cache has no copy, and the cache saves the files + # when the job ends. + - name: Fetch the free Lite data files + if: steps.cache.outputs.cache-hit != 'true' + run: ./cxx/data/get-data-files.ps1 51Degrees-LiteV4.1.hash '20000 Evidence Records.yml' '20000 User Agents.csv' 51Degrees-LiteV41.ipi 51Degrees-IPIV4AsnIpiV41.ipi + + free-threaded: + name: Python ${{ matrix.python }} on ${{ matrix.os }} + needs: free-data-files + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + python: '3.13t' + - os: ubuntu-latest + python: '3.14t' + - os: windows-latest + python: '3.14t' + runs-on: ${{ matrix.os }} + defaults: + run: + shell: pwsh + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + # The native source is the cxx submodule, and the data files are + # linked from its own data submodule. + submodules: recursive + # The package versions written below come from the release tags. + fetch-depth: 0 + + # ci/write-package-versions.ps1 works out the versions with common-ci's + # release plan. + - name: Checkout common-ci + uses: actions/checkout@v4 + with: + repository: 51Degrees/common-ci + ref: main + path: .common-ci + persist-credentials: false + + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Check the build is free-threaded + run: | + python -c "import sys, sysconfig; print(sys.version); assert sysconfig.get_config_var('Py_GIL_DISABLED') == 1, 'not a free-threaded build'; assert not sys._is_gil_enabled(), 'the GIL is on at start'" + + - name: Install the packages the tests use + env: + COMMON_CI: ${{ github.workspace }}/.common-ci + run: | + # A pip install that fails stops the step here, rather than + # leaving the tests to fail on a missing module. + $PSNativeCommandUseErrorActionPreference = $true + python -m pip install --upgrade pip + python -m pip install pytest flask flask-unittest json5 ruamel.yaml Cython setuptools + # A package installed from this checkout is 0.0.0 without the + # version.txt this writes, which is below the versions the packages + # here require of each other, so pip would refuse them. + ./ci/write-package-versions.ps1 + # The pipeline packages come from this checkout, as the tests run + # with the pipeline code they are released with. + python -m pip install ` + -e pipeline/fiftyone_pipeline_core ` + -e pipeline/fiftyone_pipeline_engines ` + -e pipeline/fiftyone_pipeline_engines_fiftyone ` + -e pipeline/fiftyone_pipeline_cloudrequestengine ` + -e device-detection/fiftyone_devicedetection_shared ` + -e device-detection/fiftyone_devicedetection_cloud ` + -e ip-intelligence/fiftyone_ipintelligence_shared + # These two require the on-premise package by name, which the + # tests read from its source folder instead. + python -m pip install --no-deps ` + -e device-detection/fiftyone_devicedetection ` + -e device-detection/fiftyone_devicedetection_examples + + - name: Restore the free Lite data files + uses: actions/cache/restore@v4 + with: + path: | + cxx/data/**/51Degrees-LiteV4.1.hash + cxx/data/**/20000 Evidence Records.yml + cxx/data/**/20000 User Agents.csv + cxx/data/**/51Degrees-LiteV41.ipi + cxx/data/**/51Degrees-IPIV4AsnIpiV41.ipi + key: ${{ needs.free-data-files.outputs.key }} + enableCrossOsArchive: true + # The first job fetched them, so a miss is a fault, never a reason + # to download them here. + fail-on-cache-miss: true + + - name: Put the IP Intelligence file where the tests look for it + run: | + New-Item -ItemType Directory -Force -Path ip-intelligence/assets | Out-Null + New-Item -ItemType HardLink ` + -Path ip-intelligence/assets/51Degrees-LiteIpiV41.ipi ` + -Target cxx/data/ip-intelligence/51Degrees-LiteV41.ipi | Out-Null + + - name: Build the native modules + run: | + if ($IsWindows) { + # A short temporary path, for the MSVC path length limit. + $env:TEMP = New-Item -ItemType Directory -Force -Path 'C:\tmp' + $env:TMP = $env:TEMP + } + foreach ($package in 'device-detection/fiftyone_devicedetection_onpremise', 'ip-intelligence/fiftyone_ipintelligence_onpremise') { + Push-Location $package + python setup.py build_clib build_ext --inplace + if ($LASTEXITCODE) { exit $LASTEXITCODE } + Pop-Location + } + + - name: Run the tests + run: | + $failed = $false + foreach ($package in 'device-detection/fiftyone_devicedetection_onpremise', 'ip-intelligence/fiftyone_ipintelligence_onpremise') { + Write-Host "::group::$package" + Push-Location $package + $env:PYTHONPATH = Join-Path $PWD 'src' + # -rs lists any test that skipped, and none should here. + python -m pytest -rs tests + if ($LASTEXITCODE) { $failed = $true } + Pop-Location + Write-Host '::endgroup::' + } + if ($failed) { exit 1 } diff --git a/ci/free-threaded-python.Tests.ps1 b/ci/free-threaded-python.Tests.ps1 new file mode 100644 index 00000000..d7028ece --- /dev/null +++ b/ci/free-threaded-python.Tests.ps1 @@ -0,0 +1,174 @@ +# Tests that .github/workflows/free-threaded-python.yml fetches the free Lite +# data files once, in one job, and that the jobs that test take them from the +# cache that job fills, rather than each downloading them again. +# The workflow is read as text, as the runners have no YAML module. + +BeforeAll { + $script:Workflow = Join-Path $PSScriptRoot '..' '.github' 'workflows' ` + 'free-threaded-python.yml' + $lines = @(Get-Content -LiteralPath $script:Workflow) + + # Each job's lines, by the job's name, from the two-space keys under jobs. + $script:Jobs = [ordered]@{} + $inJobs = $false + $name = $null + foreach ($line in $lines) { + if ($line -match '^\S') { + $inJobs = $line -match '^jobs:\s*$' + $name = $null + continue + } + if (-not $inJobs) { continue } + if ($line -match '^ ([\w-]+):\s*$') { + $name = $Matches[1] + $script:Jobs[$name] = [System.Collections.Generic.List[string]]::new() + continue + } + if ($name) { $script:Jobs[$name].Add($line) } + } + + # The text of the step that uses the action given, from its "- name:" or + # "- uses:" line to the next step. + function Get-Step([string]$Job, [string]$Uses) { + $steps = @() + $current = $null + foreach ($line in $script:Jobs[$Job]) { + if ($line -match '^ - ') { + if ($null -ne $current) { $steps += , $current } + $current = [System.Collections.Generic.List[string]]::new() + } + if ($null -ne $current) { $current.Add($line) } + } + if ($null -ne $current) { $steps += , $current } + return @($steps | Where-Object { + ($_ -join "`n") -match "uses:\s*$([regex]::Escape($Uses))@" }) + } + + # The lines of a "path: |" block in a step, trimmed. + function Get-Paths($Step) { + $paths = @() + $inPaths = $false + foreach ($line in $Step) { + if ($line -match '^\s+path:\s*\|\s*$') { $inPaths = $true; continue } + if ($inPaths -and $line -match '^\s{12,}\S') { + $paths += $line.Trim() + continue + } + $inPaths = $false + } + return $paths + } + + function Get-Value($Step, [string]$Key) { + foreach ($line in $Step) { + if ($line -match "^\s+$Key\:\s*(.+?)\s*$") { return $Matches[1] } + } + } + + $script:Fetching = @($script:Jobs.Keys | Where-Object { + ($script:Jobs[$_] -join "`n") -match 'get-data-files\.ps1' }) + $script:Testing = @($script:Jobs.Keys | Where-Object { + ($script:Jobs[$_] -join "`n") -match 'pytest' }) +} + +Describe 'free-threaded-python.yml' { + It 'downloads the free data files in one job only' { + $script:Fetching.Count | Should -Be 1 + } + + It 'downloads them in a job that is not a matrix' { + foreach ($job in $script:Fetching) { + ($script:Jobs[$job] -join "`n") | Should -Not -Match 'matrix:' + } + } + + It 'never downloads them in a job that runs the tests' { + $script:Testing.Count | Should -BeGreaterThan 0 + foreach ($job in $script:Testing) { + $script:Fetching | Should -Not -Contain $job + } + } + + It 'saves them in the cache, and downloads them only when it misses' { + $job = $script:Fetching[0] + $cache = @(Get-Step $job 'actions/cache') + $cache.Count | Should -Be 1 + Get-Value $cache[0] 'id' | Should -Not -BeNullOrEmpty + $id = Get-Value $cache[0] 'id' + $fetch = ($script:Jobs[$job] -join "`n") + $fetch | Should -Match ("if:\s*steps\.$id\.outputs\.cache-hit\s*!=\s*" + + "'true'\s*\n\s+run:\s*[|]?\s*\n?\s*\./cxx/data/get-data-files\.ps1") + } + + It 'restores them in each job that runs the tests, from the same key and paths' { + $job = $script:Fetching[0] + $save = @(Get-Step $job 'actions/cache')[0] + $keyStep = Get-Value $save 'key' + $keyStep | Should -Match '^\$\{\{\s*steps\.([\w-]+)\.outputs\.key\s*\}\}$' + ($script:Jobs[$job] -join "`n") | Should -Match ( + '\n outputs:\s*\n\s+key:\s*' + [regex]::Escape($keyStep)) + foreach ($testing in $script:Testing) { + $text = $script:Jobs[$testing] -join "`n" + $text | Should -Match "\n needs:\s*\[?\s*$([regex]::Escape($job))\b" + $restore = @(Get-Step $testing 'actions/cache/restore') + $restore.Count | Should -Be 1 + Get-Value $restore[0] 'key' | + Should -Be ('${{ needs.' + $job + '.outputs.key }}') + Get-Value $restore[0] 'fail-on-cache-miss' | Should -Be 'true' + Get-Paths $restore[0] | Should -Be (Get-Paths $save) + } + } + + It 'saves and restores across operating systems, for the Windows jobs' { + $job = $script:Fetching[0] + $steps = @(Get-Step $job 'actions/cache') + @($script:Testing | + ForEach-Object { Get-Step $_ 'actions/cache/restore' }) + $steps.Count | Should -BeGreaterThan 1 + foreach ($step in $steps) { + Get-Value $step 'enableCrossOsArchive' | Should -Be 'true' + } + } + + It 'writes the package versions before it installs the packages from this checkout' { + foreach ($testing in $script:Testing) { + $text = $script:Jobs[$testing] -join "`n" + $write = $text.IndexOf('./ci/write-package-versions.ps1') + $install = $text.IndexOf('-e pipeline/') + $write | Should -BeGreaterThan -1 + $install | Should -BeGreaterThan $write + } + } + + It 'checks out the release tags and common-ci, which the versions are worked out from' { + foreach ($testing in $script:Testing) { + $checkouts = @(Get-Step $testing 'actions/checkout') + $own = @($checkouts | Where-Object { + -not (Get-Value $_ 'repository') }) + $own.Count | Should -Be 1 + Get-Value $own[0] 'fetch-depth' | Should -Be '0' + $commonCi = @($checkouts | Where-Object { + (Get-Value $_ 'repository') -eq '51Degrees/common-ci' }) + $commonCi.Count | Should -Be 1 + $path = Get-Value $commonCi[0] 'path' + ($script:Jobs[$testing] -join "`n") | + Should -Match "COMMON_CI:.*$([regex]::Escape($path))" + } + } + + It 'fails the install step when pip fails' { + foreach ($testing in $script:Testing) { + $text = $script:Jobs[$testing] -join "`n" + $text | Should -Match ('- name: Install the packages the tests use' + + '\s*\n(\s+\S.*\n)*?\s+\$PSNativeCommandUseErrorActionPreference = \$true') + } + } + + It 'caches each of the free data files the tests use' { + $paths = Get-Paths (@(Get-Step $script:Fetching[0] 'actions/cache')[0]) + foreach ($file in '51Degrees-LiteV4.1.hash', '51Degrees-LiteV41.ipi', + '51Degrees-IPIV4AsnIpiV41.ipi', '20000 Evidence Records.yml', + '20000 User Agents.csv') { + $paths | Should -Contain "cxx/data/**/$file" + } + } +} diff --git a/device-detection/fiftyone_devicedetection_onpremise/readme.md b/device-detection/fiftyone_devicedetection_onpremise/readme.md index eb1b0178..208ce8ff 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/readme.md +++ b/device-detection/fiftyone_devicedetection_onpremise/readme.md @@ -78,6 +78,18 @@ To run the tests use: `python -m unittest discover -s tests -p test*.py -b` +## Free-threaded Python + +The package is tested on the free-threaded builds of Python 3.13 and 3.14 on Linux, and of Python 3.14 on Windows. It is published as source, so installing it on a free-threaded build needs a C and C++ compiler that builds for that Python. Its native module does not declare that it can run without the global interpreter lock (GIL), so Python turns the GIL back on for the whole process when it loads the module, and prints this warning. + +``` +RuntimeWarning: The global interpreter lock (GIL) has been enabled to load module 'fiftyone_devicedetection_onpremise._DeviceDetectionHashEngineModule', which has not declared that it can run safely without the GIL. To override this behavior and keep the GIL disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0. +``` + +From then on every thread in the program runs with the GIL, the package's native code included, as on the standard build of Python, so neither the package nor the rest of the program gains anything from free-threading. To start with the GIL on, and so without the warning, set the environment variable `PYTHON_GIL=1` or run Python with `-X gil=1`. The package has not been tested with the GIL kept off by `PYTHON_GIL=0`. + +Whichever build of Python runs it, the results of one request are not safe to use from several threads at once. Give each thread its own Flow Data, or coordinate access with a lock of your own. + ## Find out more - [Device Detection](https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=find-out-more-device-detection) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index 9982124b..955c71a6 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -64,6 +64,16 @@ class SwigData(AspectData): thread-safe access, and either gives each thread its own Flow Data, and so its own results instance, or coordinates its own access, for example with a lock of its own around every read and around close(). + + The package is tested on the free-threaded builds of Python 3.13 and + 3.14 on Linux, and of Python 3.14 on Windows. The native module does not + declare that it can run without the GIL, so Python turns the GIL back on + for the whole process when it loads the module, with a RuntimeWarning. + Every thread in the program then runs with the GIL, the native code + included, as on the standard build of Python, so nothing gains from + free-threading. Set PYTHON_GIL=1, or run Python with -X gil=1, to start + with the GIL on and so without the warning. A results instance is still + not safe to use from several threads at once. """ def __init__(self, flow_element, swig_results): diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py new file mode 100644 index 00000000..baf0fc78 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py @@ -0,0 +1,85 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +On a free-threaded build of Python, loading the native module turns the GIL +back on, with a RuntimeWarning, because the module does not declare that it +runs without the GIL. A change that loaded it with the GIL still off fails +here. Each check runs in a new interpreter, as this one may have loaded the +module already. +""" + +import os +import subprocess +import sys +import sysconfig +import unittest + +# The SWIG module, which loads the native module when it is imported. +MODULE = "fiftyone_devicedetection_onpremise.DeviceDetectionHashEngineModule" +NATIVE_MODULE = "_DeviceDetectionHashEngineModule" + +FREE_THREADED = sysconfig.get_config_var("Py_GIL_DISABLED") == 1 + +# Prints whether the GIL is on before and after the import, then the +# warnings the import gave, one to a line. +CHECK = "\n".join([ + "import sys, warnings", + "warnings.simplefilter('always')", + "before = sys._is_gil_enabled()", + "with warnings.catch_warnings(record=True) as caught:", + " import " + MODULE, + "print(before, sys._is_gil_enabled())", + "for warning in caught:", + " print(warning.category.__name__, str(warning.message))", +]) + + +@unittest.skipUnless(FREE_THREADED, "Python is not a free-threaded build") +class FreeThreadedTests(unittest.TestCase): + + def import_in_a_new_interpreter(self): + env = dict(os.environ) + # PYTHON_GIL would set the GIL whatever the module declares. + env.pop("PYTHON_GIL", None) + result = subprocess.run( + [sys.executable, "-c", CHECK], env=env, capture_output=True, + text=True) + # The new interpreter's own error says why an import failed. + self.assertEqual(0, result.returncode, result.stderr) + return result.stdout.splitlines() + + def test_loading_the_native_module_turns_the_gil_on(self): + lines = self.import_in_a_new_interpreter() + self.assertEqual("False True", lines[0]) + + def test_loading_the_native_module_warns_that_the_gil_is_on(self): + warnings = self.import_in_a_new_interpreter()[1:] + self.assertTrue( + any(line.startswith("RuntimeWarning ") + and "GIL" in line and NATIVE_MODULE in line + for line in warnings), + warnings) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md index 5cd4011b..1422561f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md @@ -56,6 +56,18 @@ flowdata.process() print(flowdata.ip.registeredname.value()) ``` +## Free-threaded Python + +The package is tested on the free-threaded builds of Python 3.13 and 3.14 on Linux, and of Python 3.14 on Windows. It is published as source, so installing it on a free-threaded build needs a C and C++ compiler that builds for that Python. Its native module does not declare that it can run without the global interpreter lock (GIL), so Python turns the GIL back on for the whole process when it loads the module, and prints this warning. + +``` +RuntimeWarning: The global interpreter lock (GIL) has been enabled to load module 'fiftyone_ipintelligence_onpremise._IpIntelligenceEngineModule', which has not declared that it can run safely without the GIL. To override this behavior and keep the GIL disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0. +``` + +From then on every thread in the program runs with the GIL, the package's native code included, as on the standard build of Python, so neither the package nor the rest of the program gains anything from free-threading. To start with the GIL on, and so without the warning, set the environment variable `PYTHON_GIL=1` or run Python with `-X gil=1`. The package has not been tested with the GIL kept off by `PYTHON_GIL=0`. + +Whichever build of Python runs it, the results of one request are not safe to use from several threads at once. Give each thread its own Flow Data, or coordinate access with a lock of your own. + ## Find out more - [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=find-out-more-ip-intelligence) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index f26cd27f..704d9a97 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -75,6 +75,16 @@ class SwigData(AspectData): thread-safe access, and either gives each thread its own Flow Data, and so its own results instance, or coordinates its own access, for example with a lock of its own around every read and around close(). + + The package is tested on the free-threaded builds of Python 3.13 and + 3.14 on Linux, and of Python 3.14 on Windows. The native module does not + declare that it can run without the GIL, so Python turns the GIL back on + for the whole process when it loads the module, with a RuntimeWarning. + Every thread in the program then runs with the GIL, the native code + included, as on the standard build of Python, so nothing gains from + free-threading. Set PYTHON_GIL=1, or run Python with -X gil=1, to start + with the GIL on and so without the warning. A results instance is still + not safe to use from several threads at once. """ def __init__(self, flow_element, swig_results, echo_ip=None, echo_ipv6=None): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py new file mode 100644 index 00000000..91253600 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py @@ -0,0 +1,85 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +On a free-threaded build of Python, loading the native module turns the GIL +back on, with a RuntimeWarning, because the module does not declare that it +runs without the GIL. A change that loaded it with the GIL still off fails +here. Each check runs in a new interpreter, as this one may have loaded the +module already. +""" + +import os +import subprocess +import sys +import sysconfig +import unittest + +# The SWIG module, which loads the native module when it is imported. +MODULE = "fiftyone_ipintelligence_onpremise.IpIntelligenceEngineModule" +NATIVE_MODULE = "_IpIntelligenceEngineModule" + +FREE_THREADED = sysconfig.get_config_var("Py_GIL_DISABLED") == 1 + +# Prints whether the GIL is on before and after the import, then the +# warnings the import gave, one to a line. +CHECK = "\n".join([ + "import sys, warnings", + "warnings.simplefilter('always')", + "before = sys._is_gil_enabled()", + "with warnings.catch_warnings(record=True) as caught:", + " import " + MODULE, + "print(before, sys._is_gil_enabled())", + "for warning in caught:", + " print(warning.category.__name__, str(warning.message))", +]) + + +@unittest.skipUnless(FREE_THREADED, "Python is not a free-threaded build") +class FreeThreadedTests(unittest.TestCase): + + def import_in_a_new_interpreter(self): + env = dict(os.environ) + # PYTHON_GIL would set the GIL whatever the module declares. + env.pop("PYTHON_GIL", None) + result = subprocess.run( + [sys.executable, "-c", CHECK], env=env, capture_output=True, + text=True) + # The new interpreter's own error says why an import failed. + self.assertEqual(0, result.returncode, result.stderr) + return result.stdout.splitlines() + + def test_loading_the_native_module_turns_the_gil_on(self): + lines = self.import_in_a_new_interpreter() + self.assertEqual("False True", lines[0]) + + def test_loading_the_native_module_warns_that_the_gil_is_on(self): + warnings = self.import_in_a_new_interpreter()[1:] + self.assertTrue( + any(line.startswith("RuntimeWarning ") + and "GIL" in line and NATIVE_MODULE in line + for line in warnings), + warnings) + + +if __name__ == "__main__": + unittest.main()