From c6883fe3cafa0b995543d485fe04421262b4a19c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 08:16:36 +0000 Subject: [PATCH 01/12] Results instances are not thread-safe, so reads take no lock SwigData in the Device Detection and IP Intelligence on-premise packages no longer holds a lock for each read of its native results or for close(). A results instance is not thread-safe, the caller is responsible for thread-safe access, and the class docstrings say so, with what to do instead. specifications#190 adds this as the "Results instances" section of the thread safety page. The thread tests asserted that reads never overlap and that close() waits for a read, which is no longer promised. They now check that the docstring says the caller coordinates threads and that an instance holds no lock. Closing results and reading afterwards is still tested with real data by test_flowdata_close_releases_results in both packages. --- .../swig_data.py | 31 ++--- .../tests/test_swig_data_threads.py | 104 +++------------- .../swig_data.py | 30 ++--- .../tests/test_swig_data_threads.py | 115 +++--------------- 4 files changed, 65 insertions(+), 215 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index 05f4cc0f..d1c5285d 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -20,8 +20,6 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -import threading - from fiftyone_pipeline_engines.aspectdata import AspectData from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue @@ -40,6 +38,16 @@ class SwigData(AspectData): """! Extention of AspectData made for retrieving results created by the SWIG wrapper of the On Premise Device Detection engine + + An instance is not safe to use from several threads at once, because it + reads its values from a results instance, the native results of one + request, which references unmanaged memory and is written into by each + read. The caller is responsible for thread-safe access. Give each thread + its own Flow Data, and so its own results instance, or hold a lock of + the caller's own around every read and around close(). No lock is taken + here, as the specification says implementations "MUST NOT lock a results + instance, or the reads that Element Data makes from one, to make it safe + for several threads". """ def __init__(self, flow_element, swig_results): @@ -58,11 +66,6 @@ def __init__(self, flow_element, swig_results): self.missing_property_service = OnPremiseMissingPropertyService() - # Reading a value writes into the native results, and the first read - # of a component can walk its graph, so one thread at a time reads - # this instance. - self._read_lock = threading.RLock() - def close(self): """! @@ -72,8 +75,7 @@ def close(self): exception. Closing the FlowData closes its results. """ - with self._read_lock: - self.swig_results = None + self.swig_results = None def get(self, key): @@ -84,13 +86,12 @@ def get(self, key): @returns mixed """ - with self._read_lock: - # Checked here, because AspectData.get turns an exception from - # get_internal into a missing property message. - if self.swig_results is None: - raise Exception(RESULTS_CLOSED) + # Checked here, because AspectData.get turns an exception from + # get_internal into a missing property message. + if self.swig_results is None: + raise Exception(RESULTS_CLOSED) - return super(SwigData, self).get(key) + return super(SwigData, self).get(key) def get_internal(self, key): diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py index e07f169d..8b93a387 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py @@ -21,59 +21,21 @@ # ********************************************************************* """ -One thread at a time reads a native results instance, because reading a value -writes into the results. +A results instance is not safe to use from several threads at once and the +caller is responsible for thread-safe access, so SwigData takes no lock and +its documentation says so. """ import threading -import time import unittest from fiftyone_devicedetection_onpremise.swig_data import SwigData -# How long a stand-in read takes, long enough that overlapping reads are seen. -READ_SECONDS = 0.02 -# The longest a test waits for another thread. -WAIT_SECONDS = 10 -# How long closing is given while a read holds the results. It must still be -# waiting at the end. -BLOCKED_SECONDS = 0.2 - -KEY = "deviceid" -VALUE = "1-2-3" - - -class OverlapCountingResults: - """Stands in for the native results and records how many reads overlap.""" - - def __init__(self): - self.guard = threading.Lock() - self.active = 0 - self.most_active = 0 - - def getDeviceId(self): - with self.guard: - self.active += 1 - self.most_active = max(self.most_active, self.active) - time.sleep(READ_SECONDS) - with self.guard: - self.active -= 1 - return "1-2-3" - - -class BlockingResults: - """Stands in for the native results with a read that waits to be let go, - so a test can act while the read is in progress.""" - - def __init__(self): - self.reading = threading.Event() - self.release = threading.Event() - - def getDeviceId(self): - self.reading.set() - self.release.wait(WAIT_SECONDS) - return "1-2-3" +# The types of the locks the threading module makes. +LOCK_TYPES = ( + type(threading.Lock()), type(threading.RLock()), + threading.Condition, threading.Semaphore) class FakeElement: @@ -83,50 +45,16 @@ class FakeElement: class SwigDataThreadTests(unittest.TestCase): - def test_reads_from_several_threads_do_not_overlap(self): - results = OverlapCountingResults() - data = SwigData(FakeElement(), results) - errors = [] - - def read(): - try: - for _ in range(3): - data.get("deviceid") - except Exception as e: - errors.append(e) - - threads = [threading.Thread(target=read) for _ in range(6)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - - self.assertEqual([], errors) - self.assertEqual(1, results.most_active) - - def test_closing_waits_for_a_read_in_progress(self): - results = BlockingResults() - data = SwigData(FakeElement(), results) - values = [] - reader = threading.Thread( - target=lambda: values.append(data.get(KEY))) - reader.start() - self.assertTrue(results.reading.wait(WAIT_SECONDS)) - - closer = threading.Thread(target=data.close) - closer.start() - closer.join(BLOCKED_SECONDS) - # The read holds the results, so closing waits for it. - self.assertTrue(closer.is_alive()) + def test_the_documentation_says_the_caller_coordinates_threads(self): + text = " ".join(SwigData.__doc__.split()) + self.assertIn("not safe to use from several threads at once", text) + self.assertIn("caller is responsible for thread-safe access", text) - results.release.set() - reader.join(WAIT_SECONDS) - closer.join(WAIT_SECONDS) - self.assertFalse(closer.is_alive()) - self.assertEqual([VALUE], [value.value() for value in values]) - self.assertIsNone(data.swig_results) - with self.assertRaises(Exception): - data.get(KEY) + def test_an_instance_holds_no_lock(self): + data = SwigData(FakeElement(), object()) + locks = [name for name, value in vars(data).items() + if isinstance(value, LOCK_TYPES)] + self.assertEqual([], locks) if __name__ == "__main__": diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index 2070a22b..90a729a4 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -21,7 +21,6 @@ # ********************************************************************* import math -import threading from decimal import Decimal, ROUND_HALF_UP from fiftyone_pipeline_engines.aspectdata import AspectData @@ -48,6 +47,16 @@ class SwigData(AspectData): """! Extension of AspectData made for retrieving results created by the SWIG wrapper of the On Premise IP Intelligence engine + + An instance is not safe to use from several threads at once, because it + reads its values from a results instance, the native results of one + request, which references unmanaged memory and is written into by each + read. The caller is responsible for thread-safe access. Give each thread + its own Flow Data, and so its own results instance, or hold a lock of + the caller's own around every read and around close(). No lock is taken + here, as the specification says implementations "MUST NOT lock a results + instance, or the reads that Element Data makes from one, to make it safe + for several threads". """ def __init__(self, flow_element, swig_results, echo_ip=None, echo_ipv6=None): @@ -72,11 +81,6 @@ def __init__(self, flow_element, swig_results, echo_ip=None, echo_ipv6=None): self.missing_property_service = OnPremiseMissingPropertyService() - # Reading a value writes into the native results, and the first read - # of a component can walk its graph, so one thread at a time reads - # this instance. - self._read_lock = threading.RLock() - def close(self): """! @@ -86,8 +90,7 @@ def close(self): exception. Closing the FlowData closes its results. """ - with self._read_lock: - self.swig_results = None + self.swig_results = None def get(self, key): @@ -98,13 +101,12 @@ def get(self, key): @returns mixed """ - with self._read_lock: - # Checked here, because AspectData.get turns an exception from - # get_internal into a missing property message. - if self.swig_results is None: - raise Exception(RESULTS_CLOSED) + # Checked here, because AspectData.get turns an exception from + # get_internal into a missing property message. + if self.swig_results is None: + raise Exception(RESULTS_CLOSED) - return super(SwigData, self).get(key) + return super(SwigData, self).get(key) @staticmethod def utf8_string_to_str(utf8_string): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py index c1476c43..7d10592f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py @@ -21,121 +21,40 @@ # ********************************************************************* """ -One thread at a time reads a native results instance, because reading a value -writes into the results. +A results instance is not safe to use from several threads at once and the +caller is responsible for thread-safe access, so SwigData takes no lock and +its documentation says so. """ import threading -import time import unittest from fiftyone_ipintelligence_onpremise.swig_data import SwigData -class Value: - - def hasValue(self): - return True - - def getValue(self): - return 5 - - -# How long a stand-in read takes, long enough that overlapping reads are seen. -READ_SECONDS = 0.02 -# The longest a test waits for another thread. -WAIT_SECONDS = 10 -# How long closing is given while a read holds the results. It must still be -# waiting at the end. -BLOCKED_SECONDS = 0.2 - -KEY = "count" -VALUE = 5 - - -class OverlapCountingResults: - """Stands in for the native results and records how many reads overlap.""" - - def __init__(self): - self.guard = threading.Lock() - self.active = 0 - self.most_active = 0 - - def getValueAsInteger(self, name): - with self.guard: - self.active += 1 - self.most_active = max(self.most_active, self.active) - time.sleep(READ_SECONDS) - with self.guard: - self.active -= 1 - return Value() - - -class BlockingResults: - """Stands in for the native results with a read that waits to be let go, - so a test can act while the read is in progress.""" - - def __init__(self): - self.reading = threading.Event() - self.release = threading.Event() - - def getValueAsInteger(self, name): - self.reading.set() - self.release.wait(WAIT_SECONDS) - return Value() +# The types of the locks the threading module makes. +LOCK_TYPES = ( + type(threading.Lock()), type(threading.RLock()), + threading.Condition, threading.Semaphore) class FakeElement: datakey = "ip" - properties = {"count": {"name": "Count", "type": "int"}} + properties = {} class SwigDataThreadTests(unittest.TestCase): - def test_reads_from_several_threads_do_not_overlap(self): - results = OverlapCountingResults() - data = SwigData(FakeElement(), results) - errors = [] - - def read(): - try: - for _ in range(3): - data.get("count") - except Exception as e: - errors.append(e) - - threads = [threading.Thread(target=read) for _ in range(6)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - - self.assertEqual([], errors) - self.assertEqual(1, results.most_active) - - def test_closing_waits_for_a_read_in_progress(self): - results = BlockingResults() - data = SwigData(FakeElement(), results) - values = [] - reader = threading.Thread( - target=lambda: values.append(data.get(KEY))) - reader.start() - self.assertTrue(results.reading.wait(WAIT_SECONDS)) - - closer = threading.Thread(target=data.close) - closer.start() - closer.join(BLOCKED_SECONDS) - # The read holds the results, so closing waits for it. - self.assertTrue(closer.is_alive()) + def test_the_documentation_says_the_caller_coordinates_threads(self): + text = " ".join(SwigData.__doc__.split()) + self.assertIn("not safe to use from several threads at once", text) + self.assertIn("caller is responsible for thread-safe access", text) - results.release.set() - reader.join(WAIT_SECONDS) - closer.join(WAIT_SECONDS) - self.assertFalse(closer.is_alive()) - self.assertEqual([VALUE], [value.value() for value in values]) - self.assertIsNone(data.swig_results) - with self.assertRaises(Exception): - data.get(KEY) + def test_an_instance_holds_no_lock(self): + data = SwigData(FakeElement(), object()) + locks = [name for name, value in vars(data).items() + if isinstance(value, LOCK_TYPES)] + self.assertEqual([], locks) if __name__ == "__main__": From 7dc2615f8750ea3fd5800e30a9a73188e982563c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 08:33:32 +0000 Subject: [PATCH 02/12] Say that Element Data read from a results instance is the exception The ElementData docstring says a finished instance is safe to read from several threads at once, as its values are Python's own managed data and do not change once written. It now also says that Element Data reading from a results instance, which holds unmanaged memory, is the exception. Multi-threaded access to a results instance is not supported, for performance reasons, so the caller gives each thread its own Flow Data or coordinates its own access. The documentation test checks both, as lines 3 to 16 of the thread safety page in specifications#190 frame it. --- .../src/fiftyone_pipeline_core/elementdata.py | 15 ++++++++++++--- .../tests/test_elementdata_documentation.py | 15 +++++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py index 5ee546d3..3fd78a7c 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py @@ -31,9 +31,18 @@ class ElementData(object): Only the Flow Element that creates an instance writes it, so an instance does not need to be safe to write from several threads. A finished instance, once the Flow Element that writes it has finished, is safe to - read from several threads at once, such as the caller's own threads. An - instance that is written from several threads, or read while another - thread writes it, is not catered for. + read from several threads at once, such as the caller's own threads, as + its values are Python's own managed data and do not change once + written. An instance that is written from several threads, or read + while another thread writes it, is not catered for. + + The exception is Element Data that reads its values from a results + instance, such as an on-premise engine's, which holds unmanaged memory. + Multi-threaded access to a results instance is not supported, for + performance reasons, so such Element Data is not safe to use from + several threads at once. The caller is responsible for thread-safe + access, and either gives each thread its own Flow Data or coordinates + its own access. """ diff --git a/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py b/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py index 58d88e28..4b00ddb1 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py @@ -2,8 +2,10 @@ """ The documentation of Element Data says it is safe to read from several threads once the Flow Element that writes it has finished, and is not catered for when -written from several threads or read while another thread writes, as the -thread safety page of the specification requires. +written from several threads or read while another thread writes. It also says +that Element Data reading from a results instance, which holds unmanaged +memory, is not safe to use from several threads at once and that the caller is +responsible, as the thread safety page of the specification requires. """ import unittest @@ -21,6 +23,15 @@ def test_the_documentation_says_what_is_safe_with_threads(self): self.assertIn("written from several threads", text) self.assertIn("read while another thread writes", text) + def test_the_documentation_says_results_instances_are_the_exception(self): + text = " ".join(ElementData.__doc__.split()) + self.assertIn("results instance", text) + self.assertIn("unmanaged memory", text) + self.assertIn( + "Multi-threaded access to a results instance is not supported, " + "for performance reasons", text) + self.assertIn("caller is responsible for thread-safe access", text) + if __name__ == "__main__": unittest.main() From 03cc1f2434183d02b49869d56f8743880b76c09c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 08:33:33 +0000 Subject: [PATCH 03/12] Say a results instance holds unmanaged memory not shared between threads Both SwigData docstrings now say a results instance holds unmanaged memory, and that multi-threaded access to it is not supported, for performance reasons, as lines 3 to 16 of the thread safety page in specifications#190 frame it. The caller either gives each thread its own Flow Data or coordinates its own access. That wording replaces the quotation of the rule against locking. The documentation tests check the new wording. --- .../swig_data.py | 17 ++++++++--------- .../tests/test_swig_data_threads.py | 4 ++++ .../swig_data.py | 17 ++++++++--------- .../tests/test_swig_data_threads.py | 4 ++++ 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index d1c5285d..c4ea2a09 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -39,15 +39,14 @@ class SwigData(AspectData): Extention of AspectData made for retrieving results created by the SWIG wrapper of the On Premise Device Detection engine - An instance is not safe to use from several threads at once, because it - reads its values from a results instance, the native results of one - request, which references unmanaged memory and is written into by each - read. The caller is responsible for thread-safe access. Give each thread - its own Flow Data, and so its own results instance, or hold a lock of - the caller's own around every read and around close(). No lock is taken - here, as the specification says implementations "MUST NOT lock a results - instance, or the reads that Element Data makes from one, to make it safe - for several threads". + An instance reads its values from a results instance, the native results + of one request, which holds unmanaged memory. Multi-threaded access to a + results instance is not supported, for performance reasons, so an + instance is not safe to use from several threads at once. The caller is + responsible for thread-safe access, and either gives each thread its own + Flow Data, and so its own results instance, or coordinates its own + access, for example with a lock of its own around every read and around + close(). """ def __init__(self, flow_element, swig_results): diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py index 8b93a387..ee191828 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py @@ -47,6 +47,10 @@ class SwigDataThreadTests(unittest.TestCase): def test_the_documentation_says_the_caller_coordinates_threads(self): text = " ".join(SwigData.__doc__.split()) + self.assertIn("unmanaged memory", text) + self.assertIn( + "Multi-threaded access to a results instance is not supported, " + "for performance reasons", text) self.assertIn("not safe to use from several threads at once", text) self.assertIn("caller is responsible for thread-safe access", text) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index 90a729a4..5285aca6 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -48,15 +48,14 @@ class SwigData(AspectData): Extension of AspectData made for retrieving results created by the SWIG wrapper of the On Premise IP Intelligence engine - An instance is not safe to use from several threads at once, because it - reads its values from a results instance, the native results of one - request, which references unmanaged memory and is written into by each - read. The caller is responsible for thread-safe access. Give each thread - its own Flow Data, and so its own results instance, or hold a lock of - the caller's own around every read and around close(). No lock is taken - here, as the specification says implementations "MUST NOT lock a results - instance, or the reads that Element Data makes from one, to make it safe - for several threads". + An instance reads its values from a results instance, the native results + of one request, which holds unmanaged memory. Multi-threaded access to a + results instance is not supported, for performance reasons, so an + instance is not safe to use from several threads at once. The caller is + responsible for thread-safe access, and either gives each thread its own + Flow Data, and so its own results instance, or coordinates its own + access, for example with a lock of its own around every read and around + close(). """ def __init__(self, flow_element, swig_results, echo_ip=None, echo_ipv6=None): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py index 7d10592f..b2d25c18 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py @@ -47,6 +47,10 @@ class SwigDataThreadTests(unittest.TestCase): def test_the_documentation_says_the_caller_coordinates_threads(self): text = " ".join(SwigData.__doc__.split()) + self.assertIn("unmanaged memory", text) + self.assertIn( + "Multi-threaded access to a results instance is not supported, " + "for performance reasons", text) self.assertIn("not safe to use from several threads at once", text) self.assertIn("caller is responsible for thread-safe access", text) From 56b501e1027b77e34427e6d37f3e9aab2a2c2606 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 08:59:04 +0000 Subject: [PATCH 04/12] Test the on-premise packages on free-threaded Python A new workflow builds the Device Detection and IP Intelligence on-premise packages on free-threaded Python 3.13 and 3.14 on Ubuntu, and 3.14 on Windows, and runs their tests with the free Lite data files. The SWIG wrappers declare that the native modules run without the GIL only when SWIGPYTHON_NOGIL is defined, and it is not, so Python turns the GIL back on when it loads them. tests/test_free_threaded.py in each package checks, in a new interpreter, that the GIL is off before the native module loads, on after, and that Python warns about it. The tests skip on every other build of Python. --- .github/workflows/free-threaded-python.yml | 126 ++++++++++++++++++ .../tests/test_free_threaded.py | 83 ++++++++++++ .../tests/test_free_threaded.py | 83 ++++++++++++ 3 files changed, 292 insertions(+) create mode 100644 .github/workflows/free-threaded-python.yml create mode 100644 device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py diff --git a/.github/workflows/free-threaded-python.yml b/.github/workflows/free-threaded-python.yml new file mode 100644 index 00000000..9c188309 --- /dev/null +++ b/.github/workflows/free-threaded-python.yml @@ -0,0 +1,126 @@ +name: Free-threaded Python + +# Builds the Device Detection and IP Intelligence on-premise packages on the +# free-threaded builds of Python and runs their tests there, with the free +# Lite data files. The native modules do not declare that they run without +# the GIL, so Python turns the GIL back on when it loads them. +# tests/test_free_threaded.py in each package checks that it does, and is +# skipped on every other build of Python. +on: + pull_request: + paths: + - 'device-detection/fiftyone_devicedetection_onpremise/**' + - 'ip-intelligence/fiftyone_ipintelligence_onpremise/**' + - 'pipeline/fiftyone_pipeline_core/**' + - 'pipeline/fiftyone_pipeline_engines/**' + - 'pipeline/fiftyone_pipeline_engines_fiftyone/**' + - 'cxx' + - '.github/workflows/free-threaded-python.yml' + push: + branches: + - main + - rc + paths: + - 'device-detection/fiftyone_devicedetection_onpremise/**' + - 'ip-intelligence/fiftyone_ipintelligence_onpremise/**' + - 'pipeline/fiftyone_pipeline_core/**' + - 'pipeline/fiftyone_pipeline_engines/**' + - 'pipeline/fiftyone_pipeline_engines_fiftyone/**' + - 'cxx' + - '.github/workflows/free-threaded-python.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + free-threaded: + name: Python ${{ matrix.python }} on ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + python: '3.13t' + - os: ubuntu-latest + python: '3.14t' + - os: windows-latest + python: '3.14t' + runs-on: ${{ matrix.os }} + defaults: + run: + shell: pwsh + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + # The native source is the cxx submodule, and the data files are + # linked from its own data submodule. + submodules: recursive + + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Check the build is free-threaded + run: | + python -c "import sys, sysconfig; print(sys.version); assert sysconfig.get_config_var('Py_GIL_DISABLED') == 1, 'not a free-threaded build'; assert not sys._is_gil_enabled(), 'the GIL is on at start'" + + - name: Install the packages the tests use + run: | + python -m pip install --upgrade pip + python -m pip install pytest flask flask-unittest json5 ruamel.yaml Cython setuptools + # The pipeline packages come from this checkout, as the tests run + # with the pipeline code they are released with. + python -m pip install ` + -e pipeline/fiftyone_pipeline_core ` + -e pipeline/fiftyone_pipeline_engines ` + -e pipeline/fiftyone_pipeline_engines_fiftyone ` + -e pipeline/fiftyone_pipeline_cloudrequestengine ` + -e device-detection/fiftyone_devicedetection_shared ` + -e device-detection/fiftyone_devicedetection_cloud ` + -e ip-intelligence/fiftyone_ipintelligence_shared + # These two require the on-premise package by name, which the + # tests read from its source folder instead. + python -m pip install --no-deps ` + -e device-detection/fiftyone_devicedetection ` + -e device-detection/fiftyone_devicedetection_examples + + - name: Fetch the free Lite data files + run: | + ./cxx/data/get-data-files.ps1 51Degrees-LiteV4.1.hash '20000 Evidence Records.yml' '20000 User Agents.csv' 51Degrees-LiteV41.ipi 51Degrees-IPIV4AsnIpiV41.ipi + New-Item -ItemType Directory -Force -Path ip-intelligence/assets | Out-Null + New-Item -ItemType HardLink ` + -Path ip-intelligence/assets/51Degrees-LiteIpiV41.ipi ` + -Target cxx/data/ip-intelligence/51Degrees-LiteV41.ipi | Out-Null + + - name: Build the native modules + run: | + if ($IsWindows) { + # A short temporary path, for the MSVC path length limit. + $env:TEMP = New-Item -ItemType Directory -Force -Path 'C:\tmp' + $env:TMP = $env:TEMP + } + foreach ($package in 'device-detection/fiftyone_devicedetection_onpremise', 'ip-intelligence/fiftyone_ipintelligence_onpremise') { + Push-Location $package + python setup.py build_clib build_ext --inplace + if ($LASTEXITCODE) { exit $LASTEXITCODE } + Pop-Location + } + + - name: Run the tests + run: | + $failed = $false + foreach ($package in 'device-detection/fiftyone_devicedetection_onpremise', 'ip-intelligence/fiftyone_ipintelligence_onpremise') { + Write-Host "::group::$package" + Push-Location $package + $env:PYTHONPATH = Join-Path $PWD 'src' + # -rs lists any test that skipped, and none should here. + python -m pytest -rs tests + if ($LASTEXITCODE) { $failed = $true } + Pop-Location + Write-Host '::endgroup::' + } + if ($failed) { exit 1 } diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py new file mode 100644 index 00000000..dd6e3228 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py @@ -0,0 +1,83 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +On a free-threaded build of Python, loading the native module turns the GIL +back on, with a RuntimeWarning, because the module does not declare that it +runs without the GIL. A change that loaded it with the GIL still off fails +here. Each check runs in a new interpreter, as this one may have loaded the +module already. +""" + +import os +import subprocess +import sys +import sysconfig +import unittest + +# The SWIG module, which loads the native module when it is imported. +MODULE = "fiftyone_devicedetection_onpremise.DeviceDetectionHashEngineModule" +NATIVE_MODULE = "_DeviceDetectionHashEngineModule" + +FREE_THREADED = sysconfig.get_config_var("Py_GIL_DISABLED") == 1 + +# Prints whether the GIL is on before and after the import, then the +# warnings the import gave, one to a line. +CHECK = "\n".join([ + "import sys, warnings", + "warnings.simplefilter('always')", + "before = sys._is_gil_enabled()", + "with warnings.catch_warnings(record=True) as caught:", + " import " + MODULE, + "print(before, sys._is_gil_enabled())", + "for warning in caught:", + " print(warning.category.__name__, str(warning.message))", +]) + + +@unittest.skipUnless(FREE_THREADED, "Python is not a free-threaded build") +class FreeThreadedTests(unittest.TestCase): + + def import_in_a_new_interpreter(self): + env = dict(os.environ) + # PYTHON_GIL would set the GIL whatever the module declares. + env.pop("PYTHON_GIL", None) + result = subprocess.run( + [sys.executable, "-c", CHECK], env=env, capture_output=True, + text=True, check=True) + return result.stdout.splitlines() + + def test_loading_the_native_module_turns_the_gil_on(self): + lines = self.import_in_a_new_interpreter() + self.assertEqual("False True", lines[0]) + + def test_loading_the_native_module_warns_that_the_gil_is_on(self): + warnings = self.import_in_a_new_interpreter()[1:] + self.assertTrue( + any(line.startswith("RuntimeWarning ") + and "GIL" in line and NATIVE_MODULE in line + for line in warnings), + warnings) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py new file mode 100644 index 00000000..b694dd50 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py @@ -0,0 +1,83 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +On a free-threaded build of Python, loading the native module turns the GIL +back on, with a RuntimeWarning, because the module does not declare that it +runs without the GIL. A change that loaded it with the GIL still off fails +here. Each check runs in a new interpreter, as this one may have loaded the +module already. +""" + +import os +import subprocess +import sys +import sysconfig +import unittest + +# The SWIG module, which loads the native module when it is imported. +MODULE = "fiftyone_ipintelligence_onpremise.IpIntelligenceEngineModule" +NATIVE_MODULE = "_IpIntelligenceEngineModule" + +FREE_THREADED = sysconfig.get_config_var("Py_GIL_DISABLED") == 1 + +# Prints whether the GIL is on before and after the import, then the +# warnings the import gave, one to a line. +CHECK = "\n".join([ + "import sys, warnings", + "warnings.simplefilter('always')", + "before = sys._is_gil_enabled()", + "with warnings.catch_warnings(record=True) as caught:", + " import " + MODULE, + "print(before, sys._is_gil_enabled())", + "for warning in caught:", + " print(warning.category.__name__, str(warning.message))", +]) + + +@unittest.skipUnless(FREE_THREADED, "Python is not a free-threaded build") +class FreeThreadedTests(unittest.TestCase): + + def import_in_a_new_interpreter(self): + env = dict(os.environ) + # PYTHON_GIL would set the GIL whatever the module declares. + env.pop("PYTHON_GIL", None) + result = subprocess.run( + [sys.executable, "-c", CHECK], env=env, capture_output=True, + text=True, check=True) + return result.stdout.splitlines() + + def test_loading_the_native_module_turns_the_gil_on(self): + lines = self.import_in_a_new_interpreter() + self.assertEqual("False True", lines[0]) + + def test_loading_the_native_module_warns_that_the_gil_is_on(self): + warnings = self.import_in_a_new_interpreter()[1:] + self.assertTrue( + any(line.startswith("RuntimeWarning ") + and "GIL" in line and NATIVE_MODULE in line + for line in warnings), + warnings) + + +if __name__ == "__main__": + unittest.main() From 424eda0ebbd753c2f3701b2e45e6e8550e20cf3d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 08:59:04 +0000 Subject: [PATCH 05/12] Say what free-threaded Python does with the on-premise packages Each on-premise readme and SwigData docstring says that free-threaded Python is supported, that loading the native module turns the GIL back on with a RuntimeWarning, and that the package therefore gains nothing from free-threading. The readmes give the warning in full, and both say how to start with the GIL on and so without the warning, which is PYTHON_GIL=1 or -X gil=1. --- .../fiftyone_devicedetection_onpremise/readme.md | 12 ++++++++++++ .../fiftyone_devicedetection_onpremise/swig_data.py | 9 +++++++++ .../fiftyone_ipintelligence_onpremise/readme.md | 12 ++++++++++++ .../fiftyone_ipintelligence_onpremise/swig_data.py | 9 +++++++++ 4 files changed, 42 insertions(+) diff --git a/device-detection/fiftyone_devicedetection_onpremise/readme.md b/device-detection/fiftyone_devicedetection_onpremise/readme.md index eb1b0178..de263c0d 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/readme.md +++ b/device-detection/fiftyone_devicedetection_onpremise/readme.md @@ -78,6 +78,18 @@ To run the tests use: `python -m unittest discover -s tests -p test*.py -b` +## Free-threaded Python + +The package runs on the free-threaded builds of Python 3.13 and 3.14. Its native module does not declare that it can run without the global interpreter lock (GIL), so Python turns the GIL back on when it loads the module and prints this warning. + +``` +RuntimeWarning: The global interpreter lock (GIL) has been enabled to load module 'fiftyone_devicedetection_onpremise._DeviceDetectionHashEngineModule', which has not declared that it can run safely without the GIL. To override this behavior and keep the GIL disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0. +``` + +The native code then runs with the GIL held, as it does on the standard build of Python, so the package gains nothing from free-threading. To start with the GIL on, and so without the warning, set the environment variable `PYTHON_GIL=1` or run Python with `-X gil=1`. The package has not been tested with the GIL kept off by `PYTHON_GIL=0`. + +Whichever build of Python runs it, the results of one request are not safe to use from several threads at once. Give each thread its own Flow Data, or coordinate access with a lock of your own. + ## Find out more - [Device Detection](https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=find-out-more-device-detection) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index c4ea2a09..d50c108c 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -47,6 +47,15 @@ class SwigData(AspectData): Flow Data, and so its own results instance, or coordinates its own access, for example with a lock of its own around every read and around close(). + + Free-threaded Python is supported. The native module does not declare + that it can run without the GIL, so Python turns the GIL back on when it + loads the module, with a RuntimeWarning. The native code then runs with + the GIL held, as it does on the standard build of Python and as it is + tested, so the package gains nothing from free-threading. Set + PYTHON_GIL=1, or run Python with -X gil=1, to start with the GIL on and + so without the warning. A results instance is still not safe to use from + several threads at once. """ def __init__(self, flow_element, swig_results): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md index 5cd4011b..ff42b01d 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md @@ -56,6 +56,18 @@ flowdata.process() print(flowdata.ip.registeredname.value()) ``` +## Free-threaded Python + +The package runs on the free-threaded builds of Python 3.13 and 3.14. Its native module does not declare that it can run without the global interpreter lock (GIL), so Python turns the GIL back on when it loads the module and prints this warning. + +``` +RuntimeWarning: The global interpreter lock (GIL) has been enabled to load module 'fiftyone_ipintelligence_onpremise._IpIntelligenceEngineModule', which has not declared that it can run safely without the GIL. To override this behavior and keep the GIL disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0. +``` + +The native code then runs with the GIL held, as it does on the standard build of Python, so the package gains nothing from free-threading. To start with the GIL on, and so without the warning, set the environment variable `PYTHON_GIL=1` or run Python with `-X gil=1`. The package has not been tested with the GIL kept off by `PYTHON_GIL=0`. + +Whichever build of Python runs it, the results of one request are not safe to use from several threads at once. Give each thread its own Flow Data, or coordinate access with a lock of your own. + ## Find out more - [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=find-out-more-ip-intelligence) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index 5285aca6..c4e8aa55 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -56,6 +56,15 @@ class SwigData(AspectData): Flow Data, and so its own results instance, or coordinates its own access, for example with a lock of its own around every read and around close(). + + Free-threaded Python is supported. The native module does not declare + that it can run without the GIL, so Python turns the GIL back on when it + loads the module, with a RuntimeWarning. The native code then runs with + the GIL held, as it does on the standard build of Python and as it is + tested, so the package gains nothing from free-threading. Set + PYTHON_GIL=1, or run Python with -X gil=1, to start with the GIL on and + so without the warning. A results instance is still not safe to use from + several threads at once. """ def __init__(self, flow_element, swig_results, echo_ip=None, echo_ipv6=None): From 6e80124706b0378c56ea26419fcc0b45d72caaa7 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 18:56:27 +0000 Subject: [PATCH 06/12] Use the specification's words for who writes Element Data and for locks The ElementData docstring now says an instance is written only by the Flow Element that provides it, or the Flow Elements that share it, and only while they process, and that the Flow Data holding it is used by one thread at a time. Element Data that reads from a results instance takes no lock to make those reads safe. Both SwigData docstrings now follow the results instance section of the thread safety page in specifications#190 at a90ba516. Reading a value writes into the results instance, and the engine, its results instances and the instance take no lock to make those reads safe for several threads. The documentation tests check the new words. --- .../swig_data.py | 18 +++++++------ .../tests/test_swig_data_threads.py | 1 + .../swig_data.py | 18 +++++++------ .../tests/test_swig_data_threads.py | 1 + .../src/fiftyone_pipeline_core/elementdata.py | 25 +++++++++---------- .../tests/test_elementdata_documentation.py | 23 +++++++++-------- 6 files changed, 46 insertions(+), 40 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index c4ea2a09..3c2204cb 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -39,14 +39,16 @@ class SwigData(AspectData): Extention of AspectData made for retrieving results created by the SWIG wrapper of the On Premise Device Detection engine - An instance reads its values from a results instance, the native results - of one request, which holds unmanaged memory. Multi-threaded access to a - results instance is not supported, for performance reasons, so an - instance is not safe to use from several threads at once. The caller is - responsible for thread-safe access, and either gives each thread its own - Flow Data, and so its own results instance, or coordinates its own - access, for example with a lock of its own around every read and around - close(). + An instance reads its values from a results instance, the object the + native code fills for one request, which references unmanaged memory. + Multi-threaded access to a results instance is not supported, for + performance reasons. Reading a value from it writes into it, so it is not + safe to use from several threads at once, and neither is this instance. + The engine, its results instances and this instance take no lock to make + those reads safe for several threads. The caller is responsible for + thread-safe access, and either gives each thread its own Flow Data, and + so its own results instance, or coordinates its own access, for example + with a lock of its own around every read and around close(). """ def __init__(self, flow_element, swig_results): diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py index ee191828..d0fa5c02 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py @@ -52,6 +52,7 @@ def test_the_documentation_says_the_caller_coordinates_threads(self): "Multi-threaded access to a results instance is not supported, " "for performance reasons", text) self.assertIn("not safe to use from several threads at once", text) + self.assertIn("take no lock", text) self.assertIn("caller is responsible for thread-safe access", text) def test_an_instance_holds_no_lock(self): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index 5285aca6..d1ddb959 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -48,14 +48,16 @@ class SwigData(AspectData): Extension of AspectData made for retrieving results created by the SWIG wrapper of the On Premise IP Intelligence engine - An instance reads its values from a results instance, the native results - of one request, which holds unmanaged memory. Multi-threaded access to a - results instance is not supported, for performance reasons, so an - instance is not safe to use from several threads at once. The caller is - responsible for thread-safe access, and either gives each thread its own - Flow Data, and so its own results instance, or coordinates its own - access, for example with a lock of its own around every read and around - close(). + An instance reads its values from a results instance, the object the + native code fills for one request, which references unmanaged memory. + Multi-threaded access to a results instance is not supported, for + performance reasons. Reading a value from it writes into it, so it is not + safe to use from several threads at once, and neither is this instance. + The engine, its results instances and this instance take no lock to make + those reads safe for several threads. The caller is responsible for + thread-safe access, and either gives each thread its own Flow Data, and + so its own results instance, or coordinates its own access, for example + with a lock of its own around every read and around close(). """ def __init__(self, flow_element, swig_results, echo_ip=None, echo_ipv6=None): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py index b2d25c18..cbbf193f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py @@ -52,6 +52,7 @@ def test_the_documentation_says_the_caller_coordinates_threads(self): "Multi-threaded access to a results instance is not supported, " "for performance reasons", text) self.assertIn("not safe to use from several threads at once", text) + self.assertIn("take no lock", text) self.assertIn("caller is responsible for thread-safe access", text) def test_an_instance_holds_no_lock(self): diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py index 3fd78a7c..008bc22b 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py @@ -28,21 +28,20 @@ class ElementData(object): Stores information created by a FlowElement based on FlowData. Stored in FlowData - Only the Flow Element that creates an instance writes it, so an instance - does not need to be safe to write from several threads. A finished - instance, once the Flow Element that writes it has finished, is safe to - read from several threads at once, such as the caller's own threads, as - its values are Python's own managed data and do not change once - written. An instance that is written from several threads, or read - while another thread writes it, is not catered for. + An instance is written only by the Flow Element that provides it, or by + the Flow Elements that share it, and only while they process. Once they + have finished, it is not written again, apart from lazy properties, so it + is safe to read from several threads at once, as its values are Python's + own managed data. The Flow Data that holds it is still used by one thread + at a time. The exception is Element Data that reads its values from a results - instance, such as an on-premise engine's, which holds unmanaged memory. - Multi-threaded access to a results instance is not supported, for - performance reasons, so such Element Data is not safe to use from - several threads at once. The caller is responsible for thread-safe - access, and either gives each thread its own Flow Data or coordinates - its own access. + instance, such as an on-premise engine's, which references unmanaged + memory. Multi-threaded access to a results instance is not supported, for + performance reasons, so such Element Data is not safe to use from several + threads at once. It takes no lock to make those reads safe. The caller is + responsible for thread-safe access, and either gives each thread its own + Flow Data or coordinates its own access. """ diff --git a/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py b/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py index 4b00ddb1..96ccbfb5 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py @@ -1,11 +1,11 @@ """ -The documentation of Element Data says it is safe to read from several threads -once the Flow Element that writes it has finished, and is not catered for when -written from several threads or read while another thread writes. It also says -that Element Data reading from a results instance, which holds unmanaged -memory, is not safe to use from several threads at once and that the caller is -responsible, as the thread safety page of the specification requires. +The documentation of Element Data says it is written only by the Flow Element +that provides it, or those that share it, and only while they process, and +that it is then safe to read from several threads. It also says that Element +Data reading from a results instance, which references unmanaged memory, is +not safe to use from several threads at once, takes no lock and leaves the +caller responsible, as the thread safety page of the specification requires. """ import unittest @@ -15,13 +15,13 @@ class ElementDataDocumentationTests(unittest.TestCase): - def test_the_documentation_says_what_is_safe_with_threads(self): + def test_the_documentation_says_who_writes_and_when(self): text = " ".join(ElementData.__doc__.split()) + self.assertIn( + "written only by the Flow Element that provides it, or by the " + "Flow Elements that share it, and only while they process", text) self.assertIn("safe to read from several threads", text) - self.assertIn("once the Flow Element that writes it has finished", text) - self.assertIn("not catered for", text) - self.assertIn("written from several threads", text) - self.assertIn("read while another thread writes", text) + self.assertIn("used by one thread at a time", text) def test_the_documentation_says_results_instances_are_the_exception(self): text = " ".join(ElementData.__doc__.split()) @@ -30,6 +30,7 @@ def test_the_documentation_says_results_instances_are_the_exception(self): self.assertIn( "Multi-threaded access to a results instance is not supported, " "for performance reasons", text) + self.assertIn("takes no lock", text) self.assertIn("caller is responsible for thread-safe access", text) From 8a079dede9fb1d4026d1e064d98a3ef507770b72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eddie=20Vereb=C3=A9lyi?= Date: Thu, 19 Nov 2026 01:21:24 +0000 Subject: [PATCH 07/12] Test that the free-threaded workflow fetches the free data files once Each job of free-threaded-python.yml downloads the free Lite data files from the Distributor on every run. The new Pester test reads the workflow and checks that one job, outside the matrix, downloads them only when the cache misses, and that every job that runs the tests restores them from that cache with the same key and paths, on Linux and Windows alike. It fails on the workflow as it stands. CI script tests now also run when the workflow changes. --- .github/workflows/ci-script-tests.yml | 6 +- ci/free-threaded-python.Tests.ps1 | 140 ++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 ci/free-threaded-python.Tests.ps1 diff --git a/.github/workflows/ci-script-tests.yml b/.github/workflows/ci-script-tests.yml index 8f68b5e0..3a74d841 100644 --- a/.github/workflows/ci-script-tests.yml +++ b/.github/workflows/ci-script-tests.yml @@ -1,13 +1,16 @@ name: CI script tests # Runs the Pester tests of the scripts in ci/, which choose what a pull request -# builds and what a publish releases. common-ci's main is checked out beside +# builds and what a publish releases, and the test of how +# free-threaded-python.yml fetches the free data files. common-ci's main is +# checked out beside # this repository, where ci/dispatch.Tests.ps1 looks for it. on: pull_request: paths: - 'ci/**' - '.github/workflows/ci-script-tests.yml' + - '.github/workflows/free-threaded-python.yml' push: branches: - main @@ -15,6 +18,7 @@ on: paths: - 'ci/**' - '.github/workflows/ci-script-tests.yml' + - '.github/workflows/free-threaded-python.yml' workflow_dispatch: permissions: diff --git a/ci/free-threaded-python.Tests.ps1 b/ci/free-threaded-python.Tests.ps1 new file mode 100644 index 00000000..19c615a8 --- /dev/null +++ b/ci/free-threaded-python.Tests.ps1 @@ -0,0 +1,140 @@ +# Tests that .github/workflows/free-threaded-python.yml fetches the free Lite +# data files once, in one job, and that the jobs that test take them from the +# cache that job fills, rather than each downloading them again. +# The workflow is read as text, as the runners have no YAML module. + +BeforeAll { + $script:Workflow = Join-Path $PSScriptRoot '..' '.github' 'workflows' ` + 'free-threaded-python.yml' + $lines = @(Get-Content -LiteralPath $script:Workflow) + + # Each job's lines, by the job's name, from the two-space keys under jobs. + $script:Jobs = [ordered]@{} + $inJobs = $false + $name = $null + foreach ($line in $lines) { + if ($line -match '^\S') { + $inJobs = $line -match '^jobs:\s*$' + $name = $null + continue + } + if (-not $inJobs) { continue } + if ($line -match '^ ([\w-]+):\s*$') { + $name = $Matches[1] + $script:Jobs[$name] = [System.Collections.Generic.List[string]]::new() + continue + } + if ($name) { $script:Jobs[$name].Add($line) } + } + + # The text of the step that uses the action given, from its "- name:" or + # "- uses:" line to the next step. + function Get-Step([string]$Job, [string]$Uses) { + $steps = @() + $current = $null + foreach ($line in $script:Jobs[$Job]) { + if ($line -match '^ - ') { + if ($current) { $steps += , $current } + $current = [System.Collections.Generic.List[string]]::new() + } + if ($current) { $current.Add($line) } + } + if ($current) { $steps += , $current } + return @($steps | Where-Object { + ($_ -join "`n") -match "uses:\s*$([regex]::Escape($Uses))@" }) + } + + # The lines of a "path: |" block in a step, trimmed. + function Get-Paths($Step) { + $paths = @() + $inPaths = $false + foreach ($line in $Step) { + if ($line -match '^\s+path:\s*\|\s*$') { $inPaths = $true; continue } + if ($inPaths -and $line -match '^\s{12,}\S') { + $paths += $line.Trim() + continue + } + $inPaths = $false + } + return $paths + } + + function Get-Value($Step, [string]$Key) { + foreach ($line in $Step) { + if ($line -match "^\s+$Key\:\s*(.+?)\s*$") { return $Matches[1] } + } + } + + $script:Fetching = @($script:Jobs.Keys | Where-Object { + ($script:Jobs[$_] -join "`n") -match 'get-data-files\.ps1' }) + $script:Testing = @($script:Jobs.Keys | Where-Object { + ($script:Jobs[$_] -join "`n") -match 'pytest' }) +} + +Describe 'free-threaded-python.yml' { + It 'downloads the free data files in one job only' { + $script:Fetching.Count | Should -Be 1 + } + + It 'downloads them in a job that is not a matrix' { + foreach ($job in $script:Fetching) { + ($script:Jobs[$job] -join "`n") | Should -Not -Match 'matrix:' + } + } + + It 'never downloads them in a job that runs the tests' { + $script:Testing.Count | Should -BeGreaterThan 0 + foreach ($job in $script:Testing) { + $script:Fetching | Should -Not -Contain $job + } + } + + It 'saves them in the cache, and downloads them only when it misses' { + $job = $script:Fetching[0] + $cache = @(Get-Step $job 'actions/cache') + $cache.Count | Should -Be 1 + Get-Value $cache[0] 'id' | Should -Not -BeNullOrEmpty + $id = Get-Value $cache[0] 'id' + $fetch = ($script:Jobs[$job] -join "`n") + $fetch | Should -Match ("if:\s*steps\.$id\.outputs\.cache-hit\s*!=\s*" + + "'true'\s*\n\s+run:\s*[|]?\s*\n?\s*\./cxx/data/get-data-files\.ps1") + } + + It 'restores them in each job that runs the tests, from the same key and paths' { + $job = $script:Fetching[0] + $save = @(Get-Step $job 'actions/cache')[0] + $keyStep = Get-Value $save 'key' + $keyStep | Should -Match '^\$\{\{\s*steps\.([\w-]+)\.outputs\.key\s*\}\}$' + ($script:Jobs[$job] -join "`n") | Should -Match ( + '\n outputs:\s*\n\s+key:\s*' + [regex]::Escape($keyStep)) + foreach ($testing in $script:Testing) { + $text = $script:Jobs[$testing] -join "`n" + $text | Should -Match "\n needs:\s*\[?\s*$([regex]::Escape($job))\b" + $restore = @(Get-Step $testing 'actions/cache/restore') + $restore.Count | Should -Be 1 + Get-Value $restore[0] 'key' | + Should -Be ('${{ needs.' + $job + '.outputs.key }}') + Get-Value $restore[0] 'fail-on-cache-miss' | Should -Be 'true' + Get-Paths $restore[0] | Should -Be (Get-Paths $save) + } + } + + It 'saves and restores across operating systems, for the Windows jobs' { + $job = $script:Fetching[0] + $steps = @(Get-Step $job 'actions/cache') + @($script:Testing | + ForEach-Object { Get-Step $_ 'actions/cache/restore' }) + $steps.Count | Should -BeGreaterThan 1 + foreach ($step in $steps) { + Get-Value $step 'enableCrossOsArchive' | Should -Be 'true' + } + } + + It 'caches each of the free data files the tests use' { + $paths = Get-Paths (@(Get-Step $script:Fetching[0] 'actions/cache')[0]) + foreach ($file in '51Degrees-LiteV4.1.hash', '51Degrees-LiteV41.ipi', + '51Degrees-IPIV4AsnIpiV41.ipi', '20000 Evidence Records.yml', + '20000 User Agents.csv') { + $paths | Should -Contain "cxx/data/**/$file" + } + } +} From d7093543edde335426dfd30e79ccf19d7fd3b047 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eddie=20Vereb=C3=A9lyi?= Date: Thu, 19 Nov 2026 01:23:55 +0000 Subject: [PATCH 08/12] Read every step of the workflow in its test The test started each step with an empty list and then added lines only when that list was not empty, which is never, so it found no step at all. It now checks for no list rather than an empty one. On the workflow as it stands it still fails six of its seven tests, now for the reasons it was written for. --- ci/free-threaded-python.Tests.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/free-threaded-python.Tests.ps1 b/ci/free-threaded-python.Tests.ps1 index 19c615a8..8a035bd4 100644 --- a/ci/free-threaded-python.Tests.ps1 +++ b/ci/free-threaded-python.Tests.ps1 @@ -34,12 +34,12 @@ BeforeAll { $current = $null foreach ($line in $script:Jobs[$Job]) { if ($line -match '^ - ') { - if ($current) { $steps += , $current } + if ($null -ne $current) { $steps += , $current } $current = [System.Collections.Generic.List[string]]::new() } - if ($current) { $current.Add($line) } + if ($null -ne $current) { $current.Add($line) } } - if ($current) { $steps += , $current } + if ($null -ne $current) { $steps += , $current } return @($steps | Where-Object { ($_ -join "`n") -match "uses:\s*$([regex]::Escape($Uses))@" }) } From 89cf923b841b54317a2735cbfab68607199bcbee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eddie=20Vereb=C3=A9lyi?= Date: Thu, 19 Nov 2026 01:23:57 +0000 Subject: [PATCH 09/12] Fetch the free data files once in the free-threaded workflow Each of the three jobs downloaded the free Lite data files from the Distributor on every run. A first job now downloads them only when the cache has no copy for the current commit of the data submodule, which links them, and the cache keeps them. The test jobs need that job and restore the files from the cache, on Linux and Windows alike, failing on a miss rather than downloading. The workflow keeps no secrets, so a pull request from a fork runs it as before. --- .github/workflows/free-threaded-python.yml | 70 +++++++++++++++++++++- 1 file changed, 67 insertions(+), 3 deletions(-) diff --git a/.github/workflows/free-threaded-python.yml b/.github/workflows/free-threaded-python.yml index 9c188309..e7820218 100644 --- a/.github/workflows/free-threaded-python.yml +++ b/.github/workflows/free-threaded-python.yml @@ -2,7 +2,10 @@ name: Free-threaded Python # Builds the Device Detection and IP Intelligence on-premise packages on the # free-threaded builds of Python and runs their tests there, with the free -# Lite data files. The native modules do not declare that they run without +# Lite data files. The first job downloads those once for each commit of the +# data submodule, which links them, and puts them in the cache. The test jobs +# restore them from there and never download them. +# ci/free-threaded-python.Tests.ps1 checks this. The native modules do not declare that they run without # the GIL, so Python turns the GIL back on when it loads them. # tests/test_free_threaded.py in each package checks that it does, and is # skipped on every other build of Python. @@ -34,8 +37,55 @@ permissions: contents: read jobs: + free-data-files: + name: Fetch the free Lite data files once + runs-on: ubuntu-latest + outputs: + key: ${{ steps.key.outputs.key }} + defaults: + run: + shell: pwsh + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + submodules: recursive + + - name: Name the cache after the data submodule + id: key + run: | + # The data submodule's .link files say which files are fetched and + # from where, so a new commit of it needs the files again. Change + # v1 when the list of files below changes. + $data = git -C cxx rev-parse HEAD:data + if ($LASTEXITCODE) { exit $LASTEXITCODE } + "key=free-lite-data-v1-$data" >> $env:GITHUB_OUTPUT + + - name: Look for the files in the cache + id: cache + uses: actions/cache@v4 + with: + path: | + cxx/data/**/51Degrees-LiteV4.1.hash + cxx/data/**/20000 Evidence Records.yml + cxx/data/**/20000 User Agents.csv + cxx/data/**/51Degrees-LiteV41.ipi + cxx/data/**/51Degrees-IPIV4AsnIpiV41.ipi + key: ${{ steps.key.outputs.key }} + # The Windows job restores what this Linux job saves. + enableCrossOsArchive: true + lookup-only: true + + # Runs only when the cache has no copy, and the cache saves the files + # when the job ends. + - name: Fetch the free Lite data files + if: steps.cache.outputs.cache-hit != 'true' + run: ./cxx/data/get-data-files.ps1 51Degrees-LiteV4.1.hash '20000 Evidence Records.yml' '20000 User Agents.csv' 51Degrees-LiteV41.ipi 51Degrees-IPIV4AsnIpiV41.ipi + free-threaded: name: Python ${{ matrix.python }} on ${{ matrix.os }} + needs: free-data-files strategy: fail-fast: false matrix: @@ -88,9 +138,23 @@ jobs: -e device-detection/fiftyone_devicedetection ` -e device-detection/fiftyone_devicedetection_examples - - name: Fetch the free Lite data files + - name: Restore the free Lite data files + uses: actions/cache/restore@v4 + with: + path: | + cxx/data/**/51Degrees-LiteV4.1.hash + cxx/data/**/20000 Evidence Records.yml + cxx/data/**/20000 User Agents.csv + cxx/data/**/51Degrees-LiteV41.ipi + cxx/data/**/51Degrees-IPIV4AsnIpiV41.ipi + key: ${{ needs.free-data-files.outputs.key }} + enableCrossOsArchive: true + # The first job fetched them, so a miss is a fault, never a reason + # to download them here. + fail-on-cache-miss: true + + - name: Put the IP Intelligence file where the tests look for it run: | - ./cxx/data/get-data-files.ps1 51Degrees-LiteV4.1.hash '20000 Evidence Records.yml' '20000 User Agents.csv' 51Degrees-LiteV41.ipi 51Degrees-IPIV4AsnIpiV41.ipi New-Item -ItemType Directory -Force -Path ip-intelligence/assets | Out-Null New-Item -ItemType HardLink ` -Path ip-intelligence/assets/51Degrees-LiteIpiV41.ipi ` From 60d16dd9562339e4f459dfe57840ed2834f91a2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eddie=20Vereb=C3=A9lyi?= Date: Thu, 19 Nov 2026 01:25:04 +0000 Subject: [PATCH 10/12] Say which free-threaded builds are tested and that the GIL is for all The readmes and SwigData docstrings said the packages run on the free-threaded builds of Python 3.13 and 3.14, and that Python turns the GIL back on when it loads the native module. They now name the builds the workflow tests (3.13 and 3.14 on Linux, 3.14 on Windows), say that the packages are published as source and so need a compiler for that Python, and say that the GIL comes back on for the whole process, so the rest of the program runs with it too. --- .../readme.md | 4 ++-- .../swig_data.py | 17 +++++++++-------- .../fiftyone_ipintelligence_onpremise/readme.md | 4 ++-- .../swig_data.py | 17 +++++++++-------- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_onpremise/readme.md b/device-detection/fiftyone_devicedetection_onpremise/readme.md index de263c0d..208ce8ff 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/readme.md +++ b/device-detection/fiftyone_devicedetection_onpremise/readme.md @@ -80,13 +80,13 @@ To run the tests use: ## Free-threaded Python -The package runs on the free-threaded builds of Python 3.13 and 3.14. Its native module does not declare that it can run without the global interpreter lock (GIL), so Python turns the GIL back on when it loads the module and prints this warning. +The package is tested on the free-threaded builds of Python 3.13 and 3.14 on Linux, and of Python 3.14 on Windows. It is published as source, so installing it on a free-threaded build needs a C and C++ compiler that builds for that Python. Its native module does not declare that it can run without the global interpreter lock (GIL), so Python turns the GIL back on for the whole process when it loads the module, and prints this warning. ``` RuntimeWarning: The global interpreter lock (GIL) has been enabled to load module 'fiftyone_devicedetection_onpremise._DeviceDetectionHashEngineModule', which has not declared that it can run safely without the GIL. To override this behavior and keep the GIL disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0. ``` -The native code then runs with the GIL held, as it does on the standard build of Python, so the package gains nothing from free-threading. To start with the GIL on, and so without the warning, set the environment variable `PYTHON_GIL=1` or run Python with `-X gil=1`. The package has not been tested with the GIL kept off by `PYTHON_GIL=0`. +From then on every thread in the program runs with the GIL, the package's native code included, as on the standard build of Python, so neither the package nor the rest of the program gains anything from free-threading. To start with the GIL on, and so without the warning, set the environment variable `PYTHON_GIL=1` or run Python with `-X gil=1`. The package has not been tested with the GIL kept off by `PYTHON_GIL=0`. Whichever build of Python runs it, the results of one request are not safe to use from several threads at once. Give each thread its own Flow Data, or coordinate access with a lock of your own. diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index d14051b9..133dd848 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -50,14 +50,15 @@ class SwigData(AspectData): so its own results instance, or coordinates its own access, for example with a lock of its own around every read and around close(). - Free-threaded Python is supported. The native module does not declare - that it can run without the GIL, so Python turns the GIL back on when it - loads the module, with a RuntimeWarning. The native code then runs with - the GIL held, as it does on the standard build of Python and as it is - tested, so the package gains nothing from free-threading. Set - PYTHON_GIL=1, or run Python with -X gil=1, to start with the GIL on and - so without the warning. A results instance is still not safe to use from - several threads at once. + The package is tested on the free-threaded builds of Python 3.13 and + 3.14 on Linux, and of Python 3.14 on Windows. The native module does not + declare that it can run without the GIL, so Python turns the GIL back on + for the whole process when it loads the module, with a RuntimeWarning. + Every thread in the program then runs with the GIL, the native code + included, as on the standard build of Python, so nothing gains from + free-threading. Set PYTHON_GIL=1, or run Python with -X gil=1, to start + with the GIL on and so without the warning. A results instance is still + not safe to use from several threads at once. """ def __init__(self, flow_element, swig_results): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md index ff42b01d..1422561f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md @@ -58,13 +58,13 @@ print(flowdata.ip.registeredname.value()) ## Free-threaded Python -The package runs on the free-threaded builds of Python 3.13 and 3.14. Its native module does not declare that it can run without the global interpreter lock (GIL), so Python turns the GIL back on when it loads the module and prints this warning. +The package is tested on the free-threaded builds of Python 3.13 and 3.14 on Linux, and of Python 3.14 on Windows. It is published as source, so installing it on a free-threaded build needs a C and C++ compiler that builds for that Python. Its native module does not declare that it can run without the global interpreter lock (GIL), so Python turns the GIL back on for the whole process when it loads the module, and prints this warning. ``` RuntimeWarning: The global interpreter lock (GIL) has been enabled to load module 'fiftyone_ipintelligence_onpremise._IpIntelligenceEngineModule', which has not declared that it can run safely without the GIL. To override this behavior and keep the GIL disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0. ``` -The native code then runs with the GIL held, as it does on the standard build of Python, so the package gains nothing from free-threading. To start with the GIL on, and so without the warning, set the environment variable `PYTHON_GIL=1` or run Python with `-X gil=1`. The package has not been tested with the GIL kept off by `PYTHON_GIL=0`. +From then on every thread in the program runs with the GIL, the package's native code included, as on the standard build of Python, so neither the package nor the rest of the program gains anything from free-threading. To start with the GIL on, and so without the warning, set the environment variable `PYTHON_GIL=1` or run Python with `-X gil=1`. The package has not been tested with the GIL kept off by `PYTHON_GIL=0`. Whichever build of Python runs it, the results of one request are not safe to use from several threads at once. Give each thread its own Flow Data, or coordinate access with a lock of your own. diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index 9ebc7f18..b5abcfa6 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -59,14 +59,15 @@ class SwigData(AspectData): so its own results instance, or coordinates its own access, for example with a lock of its own around every read and around close(). - Free-threaded Python is supported. The native module does not declare - that it can run without the GIL, so Python turns the GIL back on when it - loads the module, with a RuntimeWarning. The native code then runs with - the GIL held, as it does on the standard build of Python and as it is - tested, so the package gains nothing from free-threading. Set - PYTHON_GIL=1, or run Python with -X gil=1, to start with the GIL on and - so without the warning. A results instance is still not safe to use from - several threads at once. + The package is tested on the free-threaded builds of Python 3.13 and + 3.14 on Linux, and of Python 3.14 on Windows. The native module does not + declare that it can run without the GIL, so Python turns the GIL back on + for the whole process when it loads the module, with a RuntimeWarning. + Every thread in the program then runs with the GIL, the native code + included, as on the standard build of Python, so nothing gains from + free-threading. Set PYTHON_GIL=1, or run Python with -X gil=1, to start + with the GIL on and so without the warning. A results instance is still + not safe to use from several threads at once. """ def __init__(self, flow_element, swig_results, echo_ip=None, echo_ipv6=None): From 4d45628fb7b375868b2ba91366e875677ed283a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eddie=20Vereb=C3=A9lyi?= Date: Thu, 19 Nov 2026 01:26:03 +0000 Subject: [PATCH 11/12] Show why the native module failed to import in the free-threaded tests The tests ran the import in a new interpreter with check=True, so a failed import raised CalledProcessError with only its exit status. They now assert that the exit status is 0 with the new interpreter's error output as the message, so a missing module or a crash on import shows its cause. --- .../tests/test_free_threaded.py | 4 +++- .../tests/test_free_threaded.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py index dd6e3228..baf0fc78 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py @@ -63,7 +63,9 @@ def import_in_a_new_interpreter(self): env.pop("PYTHON_GIL", None) result = subprocess.run( [sys.executable, "-c", CHECK], env=env, capture_output=True, - text=True, check=True) + text=True) + # The new interpreter's own error says why an import failed. + self.assertEqual(0, result.returncode, result.stderr) return result.stdout.splitlines() def test_loading_the_native_module_turns_the_gil_on(self): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py index b694dd50..91253600 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py @@ -63,7 +63,9 @@ def import_in_a_new_interpreter(self): env.pop("PYTHON_GIL", None) result = subprocess.run( [sys.executable, "-c", CHECK], env=env, capture_output=True, - text=True, check=True) + text=True) + # The new interpreter's own error says why an import failed. + self.assertEqual(0, result.returncode, result.stderr) return result.stdout.splitlines() def test_loading_the_native_module_turns_the_gil_on(self): From c7a07ee654a2b8b723092eb0df7feec5a4974c2a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 21 Nov 2026 18:15:35 +0000 Subject: [PATCH 12/12] Write the package versions before the free-threaded install The packages here now require each other at a minimum version, and one installed from the checkout is 0.0.0 without a version.txt, so pip refused the editable installs and the tests then failed on missing modules. The test job now fetches the release tags, checks out common-ci, and runs ci/write-package-versions.ps1 before it installs, as the main CI does through ci/setup-environment.ps1. A pip install that fails now stops the step. ci/free-threaded-python.Tests.ps1 checks all three. --- .github/workflows/free-threaded-python.yml | 21 +++++++++++++ ci/free-threaded-python.Tests.ps1 | 34 ++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/.github/workflows/free-threaded-python.yml b/.github/workflows/free-threaded-python.yml index e7820218..d1dc2c87 100644 --- a/.github/workflows/free-threaded-python.yml +++ b/.github/workflows/free-threaded-python.yml @@ -108,6 +108,18 @@ jobs: # The native source is the cxx submodule, and the data files are # linked from its own data submodule. submodules: recursive + # The package versions written below come from the release tags. + fetch-depth: 0 + + # ci/write-package-versions.ps1 works out the versions with common-ci's + # release plan. + - name: Checkout common-ci + uses: actions/checkout@v4 + with: + repository: 51Degrees/common-ci + ref: main + path: .common-ci + persist-credentials: false - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v5 @@ -119,9 +131,18 @@ jobs: python -c "import sys, sysconfig; print(sys.version); assert sysconfig.get_config_var('Py_GIL_DISABLED') == 1, 'not a free-threaded build'; assert not sys._is_gil_enabled(), 'the GIL is on at start'" - name: Install the packages the tests use + env: + COMMON_CI: ${{ github.workspace }}/.common-ci run: | + # A pip install that fails stops the step here, rather than + # leaving the tests to fail on a missing module. + $PSNativeCommandUseErrorActionPreference = $true python -m pip install --upgrade pip python -m pip install pytest flask flask-unittest json5 ruamel.yaml Cython setuptools + # A package installed from this checkout is 0.0.0 without the + # version.txt this writes, which is below the versions the packages + # here require of each other, so pip would refuse them. + ./ci/write-package-versions.ps1 # The pipeline packages come from this checkout, as the tests run # with the pipeline code they are released with. python -m pip install ` diff --git a/ci/free-threaded-python.Tests.ps1 b/ci/free-threaded-python.Tests.ps1 index 8a035bd4..d7028ece 100644 --- a/ci/free-threaded-python.Tests.ps1 +++ b/ci/free-threaded-python.Tests.ps1 @@ -129,6 +129,40 @@ Describe 'free-threaded-python.yml' { } } + It 'writes the package versions before it installs the packages from this checkout' { + foreach ($testing in $script:Testing) { + $text = $script:Jobs[$testing] -join "`n" + $write = $text.IndexOf('./ci/write-package-versions.ps1') + $install = $text.IndexOf('-e pipeline/') + $write | Should -BeGreaterThan -1 + $install | Should -BeGreaterThan $write + } + } + + It 'checks out the release tags and common-ci, which the versions are worked out from' { + foreach ($testing in $script:Testing) { + $checkouts = @(Get-Step $testing 'actions/checkout') + $own = @($checkouts | Where-Object { + -not (Get-Value $_ 'repository') }) + $own.Count | Should -Be 1 + Get-Value $own[0] 'fetch-depth' | Should -Be '0' + $commonCi = @($checkouts | Where-Object { + (Get-Value $_ 'repository') -eq '51Degrees/common-ci' }) + $commonCi.Count | Should -Be 1 + $path = Get-Value $commonCi[0] 'path' + ($script:Jobs[$testing] -join "`n") | + Should -Match "COMMON_CI:.*$([regex]::Escape($path))" + } + } + + It 'fails the install step when pip fails' { + foreach ($testing in $script:Testing) { + $text = $script:Jobs[$testing] -join "`n" + $text | Should -Match ('- name: Install the packages the tests use' + + '\s*\n(\s+\S.*\n)*?\s+\$PSNativeCommandUseErrorActionPreference = \$true') + } + } + It 'caches each of the free data files the tests use' { $paths = Get-Paths (@(Get-Step $script:Fetching[0] 'actions/cache')[0]) foreach ($file in '51Degrees-LiteV4.1.hash', '51Degrees-LiteV41.ipi',