Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .github/workflows/ci-script-tests.yml
Original file line number Diff line number Diff line change
@@ -1,20 +1,24 @@
name: CI script tests

# Runs the Pester tests of the scripts in ci/, which choose what a pull request
# builds and what a publish releases. common-ci's main is checked out beside
# builds and what a publish releases, and the test of how
# free-threaded-python.yml fetches the free data files. common-ci's main is
# checked out beside
# this repository, where ci/dispatch.Tests.ps1 looks for it.
on:
pull_request:
paths:
- 'ci/**'
- '.github/workflows/ci-script-tests.yml'
- '.github/workflows/free-threaded-python.yml'
push:
branches:
- main
- rc
paths:
- 'ci/**'
- '.github/workflows/ci-script-tests.yml'
- '.github/workflows/free-threaded-python.yml'
workflow_dispatch:

permissions:
Expand Down
211 changes: 211 additions & 0 deletions .github/workflows/free-threaded-python.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
name: Free-threaded Python

# Builds the Device Detection and IP Intelligence on-premise packages on the
# free-threaded builds of Python and runs their tests there, with the free
# Lite data files. The first job downloads those once for each commit of the
# data submodule, which links them, and puts them in the cache. The test jobs
# restore them from there and never download them.
# ci/free-threaded-python.Tests.ps1 checks this. The native modules do not declare that they run without
# the GIL, so Python turns the GIL back on when it loads them.
# tests/test_free_threaded.py in each package checks that it does, and is
# skipped on every other build of Python.
on:
pull_request:
paths:
- 'device-detection/fiftyone_devicedetection_onpremise/**'
- 'ip-intelligence/fiftyone_ipintelligence_onpremise/**'
- 'pipeline/fiftyone_pipeline_core/**'
- 'pipeline/fiftyone_pipeline_engines/**'
- 'pipeline/fiftyone_pipeline_engines_fiftyone/**'
- 'cxx'
- '.github/workflows/free-threaded-python.yml'
push:
branches:
- main
- rc
paths:
- 'device-detection/fiftyone_devicedetection_onpremise/**'
- 'ip-intelligence/fiftyone_ipintelligence_onpremise/**'
- 'pipeline/fiftyone_pipeline_core/**'
- 'pipeline/fiftyone_pipeline_engines/**'
- 'pipeline/fiftyone_pipeline_engines_fiftyone/**'
- 'cxx'
- '.github/workflows/free-threaded-python.yml'
workflow_dispatch:

permissions:
contents: read

jobs:
free-data-files:
name: Fetch the free Lite data files once
runs-on: ubuntu-latest
outputs:
key: ${{ steps.key.outputs.key }}
defaults:
run:
shell: pwsh
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive

- name: Name the cache after the data submodule
id: key
run: |
# The data submodule's .link files say which files are fetched and
# from where, so a new commit of it needs the files again. Change
# v1 when the list of files below changes.
$data = git -C cxx rev-parse HEAD:data
if ($LASTEXITCODE) { exit $LASTEXITCODE }
"key=free-lite-data-v1-$data" >> $env:GITHUB_OUTPUT

- name: Look for the files in the cache
id: cache
uses: actions/cache@v4
with:
path: |
cxx/data/**/51Degrees-LiteV4.1.hash
cxx/data/**/20000 Evidence Records.yml
cxx/data/**/20000 User Agents.csv
cxx/data/**/51Degrees-LiteV41.ipi
cxx/data/**/51Degrees-IPIV4AsnIpiV41.ipi
key: ${{ steps.key.outputs.key }}
# The Windows job restores what this Linux job saves.
enableCrossOsArchive: true
lookup-only: true

# Runs only when the cache has no copy, and the cache saves the files
# when the job ends.
- name: Fetch the free Lite data files
if: steps.cache.outputs.cache-hit != 'true'
run: ./cxx/data/get-data-files.ps1 51Degrees-LiteV4.1.hash '20000 Evidence Records.yml' '20000 User Agents.csv' 51Degrees-LiteV41.ipi 51Degrees-IPIV4AsnIpiV41.ipi

free-threaded:
name: Python ${{ matrix.python }} on ${{ matrix.os }}
needs: free-data-files
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
python: '3.13t'
- os: ubuntu-latest
python: '3.14t'
- os: windows-latest
python: '3.14t'
runs-on: ${{ matrix.os }}
defaults:
run:
shell: pwsh
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
persist-credentials: false
# The native source is the cxx submodule, and the data files are
# linked from its own data submodule.
submodules: recursive
# The package versions written below come from the release tags.
fetch-depth: 0

# ci/write-package-versions.ps1 works out the versions with common-ci's
# release plan.
- name: Checkout common-ci
uses: actions/checkout@v4
with:
repository: 51Degrees/common-ci
ref: main
path: .common-ci
persist-credentials: false

- name: Set up Python ${{ matrix.python }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}

- name: Check the build is free-threaded
run: |
python -c "import sys, sysconfig; print(sys.version); assert sysconfig.get_config_var('Py_GIL_DISABLED') == 1, 'not a free-threaded build'; assert not sys._is_gil_enabled(), 'the GIL is on at start'"

- name: Install the packages the tests use
env:
COMMON_CI: ${{ github.workspace }}/.common-ci
run: |
# A pip install that fails stops the step here, rather than
# leaving the tests to fail on a missing module.
$PSNativeCommandUseErrorActionPreference = $true
python -m pip install --upgrade pip
python -m pip install pytest flask flask-unittest json5 ruamel.yaml Cython setuptools
# A package installed from this checkout is 0.0.0 without the
# version.txt this writes, which is below the versions the packages
# here require of each other, so pip would refuse them.
./ci/write-package-versions.ps1
# The pipeline packages come from this checkout, as the tests run
# with the pipeline code they are released with.
python -m pip install `
-e pipeline/fiftyone_pipeline_core `
-e pipeline/fiftyone_pipeline_engines `
-e pipeline/fiftyone_pipeline_engines_fiftyone `
-e pipeline/fiftyone_pipeline_cloudrequestengine `
-e device-detection/fiftyone_devicedetection_shared `
-e device-detection/fiftyone_devicedetection_cloud `
-e ip-intelligence/fiftyone_ipintelligence_shared
# These two require the on-premise package by name, which the
# tests read from its source folder instead.
python -m pip install --no-deps `
-e device-detection/fiftyone_devicedetection `
-e device-detection/fiftyone_devicedetection_examples

- name: Restore the free Lite data files
uses: actions/cache/restore@v4
with:
path: |
cxx/data/**/51Degrees-LiteV4.1.hash
cxx/data/**/20000 Evidence Records.yml
cxx/data/**/20000 User Agents.csv
cxx/data/**/51Degrees-LiteV41.ipi
cxx/data/**/51Degrees-IPIV4AsnIpiV41.ipi
key: ${{ needs.free-data-files.outputs.key }}
enableCrossOsArchive: true
# The first job fetched them, so a miss is a fault, never a reason
# to download them here.
fail-on-cache-miss: true

- name: Put the IP Intelligence file where the tests look for it
run: |
New-Item -ItemType Directory -Force -Path ip-intelligence/assets | Out-Null
New-Item -ItemType HardLink `
-Path ip-intelligence/assets/51Degrees-LiteIpiV41.ipi `
-Target cxx/data/ip-intelligence/51Degrees-LiteV41.ipi | Out-Null

- name: Build the native modules
run: |
if ($IsWindows) {
# A short temporary path, for the MSVC path length limit.
$env:TEMP = New-Item -ItemType Directory -Force -Path 'C:\tmp'
$env:TMP = $env:TEMP
}
foreach ($package in 'device-detection/fiftyone_devicedetection_onpremise', 'ip-intelligence/fiftyone_ipintelligence_onpremise') {
Push-Location $package
python setup.py build_clib build_ext --inplace
if ($LASTEXITCODE) { exit $LASTEXITCODE }
Pop-Location
}

- name: Run the tests
run: |
$failed = $false
foreach ($package in 'device-detection/fiftyone_devicedetection_onpremise', 'ip-intelligence/fiftyone_ipintelligence_onpremise') {
Write-Host "::group::$package"
Push-Location $package
$env:PYTHONPATH = Join-Path $PWD 'src'
# -rs lists any test that skipped, and none should here.
python -m pytest -rs tests
if ($LASTEXITCODE) { $failed = $true }
Pop-Location
Write-Host '::endgroup::'
}
if ($failed) { exit 1 }
Loading
Loading