From c6883fe3cafa0b995543d485fe04421262b4a19c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 08:16:36 +0000 Subject: [PATCH 1/5] Results instances are not thread-safe, so reads take no lock SwigData in the Device Detection and IP Intelligence on-premise packages no longer holds a lock for each read of its native results or for close(). A results instance is not thread-safe, the caller is responsible for thread-safe access, and the class docstrings say so, with what to do instead. specifications#190 adds this as the "Results instances" section of the thread safety page. The thread tests asserted that reads never overlap and that close() waits for a read, which is no longer promised. They now check that the docstring says the caller coordinates threads and that an instance holds no lock. Closing results and reading afterwards is still tested with real data by test_flowdata_close_releases_results in both packages. --- .../swig_data.py | 31 ++--- .../tests/test_swig_data_threads.py | 104 +++------------- .../swig_data.py | 30 ++--- .../tests/test_swig_data_threads.py | 115 +++--------------- 4 files changed, 65 insertions(+), 215 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index 05f4cc0f..d1c5285d 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -20,8 +20,6 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -import threading - from fiftyone_pipeline_engines.aspectdata import AspectData from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue @@ -40,6 +38,16 @@ class SwigData(AspectData): """! Extention of AspectData made for retrieving results created by the SWIG wrapper of the On Premise Device Detection engine + + An instance is not safe to use from several threads at once, because it + reads its values from a results instance, the native results of one + request, which references unmanaged memory and is written into by each + read. The caller is responsible for thread-safe access. Give each thread + its own Flow Data, and so its own results instance, or hold a lock of + the caller's own around every read and around close(). No lock is taken + here, as the specification says implementations "MUST NOT lock a results + instance, or the reads that Element Data makes from one, to make it safe + for several threads". """ def __init__(self, flow_element, swig_results): @@ -58,11 +66,6 @@ def __init__(self, flow_element, swig_results): self.missing_property_service = OnPremiseMissingPropertyService() - # Reading a value writes into the native results, and the first read - # of a component can walk its graph, so one thread at a time reads - # this instance. - self._read_lock = threading.RLock() - def close(self): """! @@ -72,8 +75,7 @@ def close(self): exception. Closing the FlowData closes its results. """ - with self._read_lock: - self.swig_results = None + self.swig_results = None def get(self, key): @@ -84,13 +86,12 @@ def get(self, key): @returns mixed """ - with self._read_lock: - # Checked here, because AspectData.get turns an exception from - # get_internal into a missing property message. - if self.swig_results is None: - raise Exception(RESULTS_CLOSED) + # Checked here, because AspectData.get turns an exception from + # get_internal into a missing property message. + if self.swig_results is None: + raise Exception(RESULTS_CLOSED) - return super(SwigData, self).get(key) + return super(SwigData, self).get(key) def get_internal(self, key): diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py index e07f169d..8b93a387 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py @@ -21,59 +21,21 @@ # ********************************************************************* """ -One thread at a time reads a native results instance, because reading a value -writes into the results. +A results instance is not safe to use from several threads at once and the +caller is responsible for thread-safe access, so SwigData takes no lock and +its documentation says so. """ import threading -import time import unittest from fiftyone_devicedetection_onpremise.swig_data import SwigData -# How long a stand-in read takes, long enough that overlapping reads are seen. -READ_SECONDS = 0.02 -# The longest a test waits for another thread. -WAIT_SECONDS = 10 -# How long closing is given while a read holds the results. It must still be -# waiting at the end. -BLOCKED_SECONDS = 0.2 - -KEY = "deviceid" -VALUE = "1-2-3" - - -class OverlapCountingResults: - """Stands in for the native results and records how many reads overlap.""" - - def __init__(self): - self.guard = threading.Lock() - self.active = 0 - self.most_active = 0 - - def getDeviceId(self): - with self.guard: - self.active += 1 - self.most_active = max(self.most_active, self.active) - time.sleep(READ_SECONDS) - with self.guard: - self.active -= 1 - return "1-2-3" - - -class BlockingResults: - """Stands in for the native results with a read that waits to be let go, - so a test can act while the read is in progress.""" - - def __init__(self): - self.reading = threading.Event() - self.release = threading.Event() - - def getDeviceId(self): - self.reading.set() - self.release.wait(WAIT_SECONDS) - return "1-2-3" +# The types of the locks the threading module makes. +LOCK_TYPES = ( + type(threading.Lock()), type(threading.RLock()), + threading.Condition, threading.Semaphore) class FakeElement: @@ -83,50 +45,16 @@ class FakeElement: class SwigDataThreadTests(unittest.TestCase): - def test_reads_from_several_threads_do_not_overlap(self): - results = OverlapCountingResults() - data = SwigData(FakeElement(), results) - errors = [] - - def read(): - try: - for _ in range(3): - data.get("deviceid") - except Exception as e: - errors.append(e) - - threads = [threading.Thread(target=read) for _ in range(6)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - - self.assertEqual([], errors) - self.assertEqual(1, results.most_active) - - def test_closing_waits_for_a_read_in_progress(self): - results = BlockingResults() - data = SwigData(FakeElement(), results) - values = [] - reader = threading.Thread( - target=lambda: values.append(data.get(KEY))) - reader.start() - self.assertTrue(results.reading.wait(WAIT_SECONDS)) - - closer = threading.Thread(target=data.close) - closer.start() - closer.join(BLOCKED_SECONDS) - # The read holds the results, so closing waits for it. - self.assertTrue(closer.is_alive()) + def test_the_documentation_says_the_caller_coordinates_threads(self): + text = " ".join(SwigData.__doc__.split()) + self.assertIn("not safe to use from several threads at once", text) + self.assertIn("caller is responsible for thread-safe access", text) - results.release.set() - reader.join(WAIT_SECONDS) - closer.join(WAIT_SECONDS) - self.assertFalse(closer.is_alive()) - self.assertEqual([VALUE], [value.value() for value in values]) - self.assertIsNone(data.swig_results) - with self.assertRaises(Exception): - data.get(KEY) + def test_an_instance_holds_no_lock(self): + data = SwigData(FakeElement(), object()) + locks = [name for name, value in vars(data).items() + if isinstance(value, LOCK_TYPES)] + self.assertEqual([], locks) if __name__ == "__main__": diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index 2070a22b..90a729a4 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -21,7 +21,6 @@ # ********************************************************************* import math -import threading from decimal import Decimal, ROUND_HALF_UP from fiftyone_pipeline_engines.aspectdata import AspectData @@ -48,6 +47,16 @@ class SwigData(AspectData): """! Extension of AspectData made for retrieving results created by the SWIG wrapper of the On Premise IP Intelligence engine + + An instance is not safe to use from several threads at once, because it + reads its values from a results instance, the native results of one + request, which references unmanaged memory and is written into by each + read. The caller is responsible for thread-safe access. Give each thread + its own Flow Data, and so its own results instance, or hold a lock of + the caller's own around every read and around close(). No lock is taken + here, as the specification says implementations "MUST NOT lock a results + instance, or the reads that Element Data makes from one, to make it safe + for several threads". """ def __init__(self, flow_element, swig_results, echo_ip=None, echo_ipv6=None): @@ -72,11 +81,6 @@ def __init__(self, flow_element, swig_results, echo_ip=None, echo_ipv6=None): self.missing_property_service = OnPremiseMissingPropertyService() - # Reading a value writes into the native results, and the first read - # of a component can walk its graph, so one thread at a time reads - # this instance. - self._read_lock = threading.RLock() - def close(self): """! @@ -86,8 +90,7 @@ def close(self): exception. Closing the FlowData closes its results. """ - with self._read_lock: - self.swig_results = None + self.swig_results = None def get(self, key): @@ -98,13 +101,12 @@ def get(self, key): @returns mixed """ - with self._read_lock: - # Checked here, because AspectData.get turns an exception from - # get_internal into a missing property message. - if self.swig_results is None: - raise Exception(RESULTS_CLOSED) + # Checked here, because AspectData.get turns an exception from + # get_internal into a missing property message. + if self.swig_results is None: + raise Exception(RESULTS_CLOSED) - return super(SwigData, self).get(key) + return super(SwigData, self).get(key) @staticmethod def utf8_string_to_str(utf8_string): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py index c1476c43..7d10592f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py @@ -21,121 +21,40 @@ # ********************************************************************* """ -One thread at a time reads a native results instance, because reading a value -writes into the results. +A results instance is not safe to use from several threads at once and the +caller is responsible for thread-safe access, so SwigData takes no lock and +its documentation says so. """ import threading -import time import unittest from fiftyone_ipintelligence_onpremise.swig_data import SwigData -class Value: - - def hasValue(self): - return True - - def getValue(self): - return 5 - - -# How long a stand-in read takes, long enough that overlapping reads are seen. -READ_SECONDS = 0.02 -# The longest a test waits for another thread. -WAIT_SECONDS = 10 -# How long closing is given while a read holds the results. It must still be -# waiting at the end. -BLOCKED_SECONDS = 0.2 - -KEY = "count" -VALUE = 5 - - -class OverlapCountingResults: - """Stands in for the native results and records how many reads overlap.""" - - def __init__(self): - self.guard = threading.Lock() - self.active = 0 - self.most_active = 0 - - def getValueAsInteger(self, name): - with self.guard: - self.active += 1 - self.most_active = max(self.most_active, self.active) - time.sleep(READ_SECONDS) - with self.guard: - self.active -= 1 - return Value() - - -class BlockingResults: - """Stands in for the native results with a read that waits to be let go, - so a test can act while the read is in progress.""" - - def __init__(self): - self.reading = threading.Event() - self.release = threading.Event() - - def getValueAsInteger(self, name): - self.reading.set() - self.release.wait(WAIT_SECONDS) - return Value() +# The types of the locks the threading module makes. +LOCK_TYPES = ( + type(threading.Lock()), type(threading.RLock()), + threading.Condition, threading.Semaphore) class FakeElement: datakey = "ip" - properties = {"count": {"name": "Count", "type": "int"}} + properties = {} class SwigDataThreadTests(unittest.TestCase): - def test_reads_from_several_threads_do_not_overlap(self): - results = OverlapCountingResults() - data = SwigData(FakeElement(), results) - errors = [] - - def read(): - try: - for _ in range(3): - data.get("count") - except Exception as e: - errors.append(e) - - threads = [threading.Thread(target=read) for _ in range(6)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - - self.assertEqual([], errors) - self.assertEqual(1, results.most_active) - - def test_closing_waits_for_a_read_in_progress(self): - results = BlockingResults() - data = SwigData(FakeElement(), results) - values = [] - reader = threading.Thread( - target=lambda: values.append(data.get(KEY))) - reader.start() - self.assertTrue(results.reading.wait(WAIT_SECONDS)) - - closer = threading.Thread(target=data.close) - closer.start() - closer.join(BLOCKED_SECONDS) - # The read holds the results, so closing waits for it. - self.assertTrue(closer.is_alive()) + def test_the_documentation_says_the_caller_coordinates_threads(self): + text = " ".join(SwigData.__doc__.split()) + self.assertIn("not safe to use from several threads at once", text) + self.assertIn("caller is responsible for thread-safe access", text) - results.release.set() - reader.join(WAIT_SECONDS) - closer.join(WAIT_SECONDS) - self.assertFalse(closer.is_alive()) - self.assertEqual([VALUE], [value.value() for value in values]) - self.assertIsNone(data.swig_results) - with self.assertRaises(Exception): - data.get(KEY) + def test_an_instance_holds_no_lock(self): + data = SwigData(FakeElement(), object()) + locks = [name for name, value in vars(data).items() + if isinstance(value, LOCK_TYPES)] + self.assertEqual([], locks) if __name__ == "__main__": From 7dc2615f8750ea3fd5800e30a9a73188e982563c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 08:33:32 +0000 Subject: [PATCH 2/5] Say that Element Data read from a results instance is the exception The ElementData docstring says a finished instance is safe to read from several threads at once, as its values are Python's own managed data and do not change once written. It now also says that Element Data reading from a results instance, which holds unmanaged memory, is the exception. Multi-threaded access to a results instance is not supported, for performance reasons, so the caller gives each thread its own Flow Data or coordinates its own access. The documentation test checks both, as lines 3 to 16 of the thread safety page in specifications#190 frame it. --- .../src/fiftyone_pipeline_core/elementdata.py | 15 ++++++++++++--- .../tests/test_elementdata_documentation.py | 15 +++++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py index 5ee546d3..3fd78a7c 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py @@ -31,9 +31,18 @@ class ElementData(object): Only the Flow Element that creates an instance writes it, so an instance does not need to be safe to write from several threads. A finished instance, once the Flow Element that writes it has finished, is safe to - read from several threads at once, such as the caller's own threads. An - instance that is written from several threads, or read while another - thread writes it, is not catered for. + read from several threads at once, such as the caller's own threads, as + its values are Python's own managed data and do not change once + written. An instance that is written from several threads, or read + while another thread writes it, is not catered for. + + The exception is Element Data that reads its values from a results + instance, such as an on-premise engine's, which holds unmanaged memory. + Multi-threaded access to a results instance is not supported, for + performance reasons, so such Element Data is not safe to use from + several threads at once. The caller is responsible for thread-safe + access, and either gives each thread its own Flow Data or coordinates + its own access. """ diff --git a/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py b/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py index 58d88e28..4b00ddb1 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py @@ -2,8 +2,10 @@ """ The documentation of Element Data says it is safe to read from several threads once the Flow Element that writes it has finished, and is not catered for when -written from several threads or read while another thread writes, as the -thread safety page of the specification requires. +written from several threads or read while another thread writes. It also says +that Element Data reading from a results instance, which holds unmanaged +memory, is not safe to use from several threads at once and that the caller is +responsible, as the thread safety page of the specification requires. """ import unittest @@ -21,6 +23,15 @@ def test_the_documentation_says_what_is_safe_with_threads(self): self.assertIn("written from several threads", text) self.assertIn("read while another thread writes", text) + def test_the_documentation_says_results_instances_are_the_exception(self): + text = " ".join(ElementData.__doc__.split()) + self.assertIn("results instance", text) + self.assertIn("unmanaged memory", text) + self.assertIn( + "Multi-threaded access to a results instance is not supported, " + "for performance reasons", text) + self.assertIn("caller is responsible for thread-safe access", text) + if __name__ == "__main__": unittest.main() From 03cc1f2434183d02b49869d56f8743880b76c09c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 08:33:33 +0000 Subject: [PATCH 3/5] Say a results instance holds unmanaged memory not shared between threads Both SwigData docstrings now say a results instance holds unmanaged memory, and that multi-threaded access to it is not supported, for performance reasons, as lines 3 to 16 of the thread safety page in specifications#190 frame it. The caller either gives each thread its own Flow Data or coordinates its own access. That wording replaces the quotation of the rule against locking. The documentation tests check the new wording. --- .../swig_data.py | 17 ++++++++--------- .../tests/test_swig_data_threads.py | 4 ++++ .../swig_data.py | 17 ++++++++--------- .../tests/test_swig_data_threads.py | 4 ++++ 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index d1c5285d..c4ea2a09 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -39,15 +39,14 @@ class SwigData(AspectData): Extention of AspectData made for retrieving results created by the SWIG wrapper of the On Premise Device Detection engine - An instance is not safe to use from several threads at once, because it - reads its values from a results instance, the native results of one - request, which references unmanaged memory and is written into by each - read. The caller is responsible for thread-safe access. Give each thread - its own Flow Data, and so its own results instance, or hold a lock of - the caller's own around every read and around close(). No lock is taken - here, as the specification says implementations "MUST NOT lock a results - instance, or the reads that Element Data makes from one, to make it safe - for several threads". + An instance reads its values from a results instance, the native results + of one request, which holds unmanaged memory. Multi-threaded access to a + results instance is not supported, for performance reasons, so an + instance is not safe to use from several threads at once. The caller is + responsible for thread-safe access, and either gives each thread its own + Flow Data, and so its own results instance, or coordinates its own + access, for example with a lock of its own around every read and around + close(). """ def __init__(self, flow_element, swig_results): diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py index 8b93a387..ee191828 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py @@ -47,6 +47,10 @@ class SwigDataThreadTests(unittest.TestCase): def test_the_documentation_says_the_caller_coordinates_threads(self): text = " ".join(SwigData.__doc__.split()) + self.assertIn("unmanaged memory", text) + self.assertIn( + "Multi-threaded access to a results instance is not supported, " + "for performance reasons", text) self.assertIn("not safe to use from several threads at once", text) self.assertIn("caller is responsible for thread-safe access", text) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index 90a729a4..5285aca6 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -48,15 +48,14 @@ class SwigData(AspectData): Extension of AspectData made for retrieving results created by the SWIG wrapper of the On Premise IP Intelligence engine - An instance is not safe to use from several threads at once, because it - reads its values from a results instance, the native results of one - request, which references unmanaged memory and is written into by each - read. The caller is responsible for thread-safe access. Give each thread - its own Flow Data, and so its own results instance, or hold a lock of - the caller's own around every read and around close(). No lock is taken - here, as the specification says implementations "MUST NOT lock a results - instance, or the reads that Element Data makes from one, to make it safe - for several threads". + An instance reads its values from a results instance, the native results + of one request, which holds unmanaged memory. Multi-threaded access to a + results instance is not supported, for performance reasons, so an + instance is not safe to use from several threads at once. The caller is + responsible for thread-safe access, and either gives each thread its own + Flow Data, and so its own results instance, or coordinates its own + access, for example with a lock of its own around every read and around + close(). """ def __init__(self, flow_element, swig_results, echo_ip=None, echo_ipv6=None): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py index 7d10592f..b2d25c18 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py @@ -47,6 +47,10 @@ class SwigDataThreadTests(unittest.TestCase): def test_the_documentation_says_the_caller_coordinates_threads(self): text = " ".join(SwigData.__doc__.split()) + self.assertIn("unmanaged memory", text) + self.assertIn( + "Multi-threaded access to a results instance is not supported, " + "for performance reasons", text) self.assertIn("not safe to use from several threads at once", text) self.assertIn("caller is responsible for thread-safe access", text) From 6e80124706b0378c56ea26419fcc0b45d72caaa7 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 18:56:27 +0000 Subject: [PATCH 4/5] Use the specification's words for who writes Element Data and for locks The ElementData docstring now says an instance is written only by the Flow Element that provides it, or the Flow Elements that share it, and only while they process, and that the Flow Data holding it is used by one thread at a time. Element Data that reads from a results instance takes no lock to make those reads safe. Both SwigData docstrings now follow the results instance section of the thread safety page in specifications#190 at a90ba516. Reading a value writes into the results instance, and the engine, its results instances and the instance take no lock to make those reads safe for several threads. The documentation tests check the new words. --- .../swig_data.py | 18 +++++++------ .../tests/test_swig_data_threads.py | 1 + .../swig_data.py | 18 +++++++------ .../tests/test_swig_data_threads.py | 1 + .../src/fiftyone_pipeline_core/elementdata.py | 25 +++++++++---------- .../tests/test_elementdata_documentation.py | 23 +++++++++-------- 6 files changed, 46 insertions(+), 40 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index c4ea2a09..3c2204cb 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -39,14 +39,16 @@ class SwigData(AspectData): Extention of AspectData made for retrieving results created by the SWIG wrapper of the On Premise Device Detection engine - An instance reads its values from a results instance, the native results - of one request, which holds unmanaged memory. Multi-threaded access to a - results instance is not supported, for performance reasons, so an - instance is not safe to use from several threads at once. The caller is - responsible for thread-safe access, and either gives each thread its own - Flow Data, and so its own results instance, or coordinates its own - access, for example with a lock of its own around every read and around - close(). + An instance reads its values from a results instance, the object the + native code fills for one request, which references unmanaged memory. + Multi-threaded access to a results instance is not supported, for + performance reasons. Reading a value from it writes into it, so it is not + safe to use from several threads at once, and neither is this instance. + The engine, its results instances and this instance take no lock to make + those reads safe for several threads. The caller is responsible for + thread-safe access, and either gives each thread its own Flow Data, and + so its own results instance, or coordinates its own access, for example + with a lock of its own around every read and around close(). """ def __init__(self, flow_element, swig_results): diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py index ee191828..d0fa5c02 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py @@ -52,6 +52,7 @@ def test_the_documentation_says_the_caller_coordinates_threads(self): "Multi-threaded access to a results instance is not supported, " "for performance reasons", text) self.assertIn("not safe to use from several threads at once", text) + self.assertIn("take no lock", text) self.assertIn("caller is responsible for thread-safe access", text) def test_an_instance_holds_no_lock(self): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index 5285aca6..d1ddb959 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -48,14 +48,16 @@ class SwigData(AspectData): Extension of AspectData made for retrieving results created by the SWIG wrapper of the On Premise IP Intelligence engine - An instance reads its values from a results instance, the native results - of one request, which holds unmanaged memory. Multi-threaded access to a - results instance is not supported, for performance reasons, so an - instance is not safe to use from several threads at once. The caller is - responsible for thread-safe access, and either gives each thread its own - Flow Data, and so its own results instance, or coordinates its own - access, for example with a lock of its own around every read and around - close(). + An instance reads its values from a results instance, the object the + native code fills for one request, which references unmanaged memory. + Multi-threaded access to a results instance is not supported, for + performance reasons. Reading a value from it writes into it, so it is not + safe to use from several threads at once, and neither is this instance. + The engine, its results instances and this instance take no lock to make + those reads safe for several threads. The caller is responsible for + thread-safe access, and either gives each thread its own Flow Data, and + so its own results instance, or coordinates its own access, for example + with a lock of its own around every read and around close(). """ def __init__(self, flow_element, swig_results, echo_ip=None, echo_ipv6=None): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py index b2d25c18..cbbf193f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py @@ -52,6 +52,7 @@ def test_the_documentation_says_the_caller_coordinates_threads(self): "Multi-threaded access to a results instance is not supported, " "for performance reasons", text) self.assertIn("not safe to use from several threads at once", text) + self.assertIn("take no lock", text) self.assertIn("caller is responsible for thread-safe access", text) def test_an_instance_holds_no_lock(self): diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py index 3fd78a7c..008bc22b 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py @@ -28,21 +28,20 @@ class ElementData(object): Stores information created by a FlowElement based on FlowData. Stored in FlowData - Only the Flow Element that creates an instance writes it, so an instance - does not need to be safe to write from several threads. A finished - instance, once the Flow Element that writes it has finished, is safe to - read from several threads at once, such as the caller's own threads, as - its values are Python's own managed data and do not change once - written. An instance that is written from several threads, or read - while another thread writes it, is not catered for. + An instance is written only by the Flow Element that provides it, or by + the Flow Elements that share it, and only while they process. Once they + have finished, it is not written again, apart from lazy properties, so it + is safe to read from several threads at once, as its values are Python's + own managed data. The Flow Data that holds it is still used by one thread + at a time. The exception is Element Data that reads its values from a results - instance, such as an on-premise engine's, which holds unmanaged memory. - Multi-threaded access to a results instance is not supported, for - performance reasons, so such Element Data is not safe to use from - several threads at once. The caller is responsible for thread-safe - access, and either gives each thread its own Flow Data or coordinates - its own access. + instance, such as an on-premise engine's, which references unmanaged + memory. Multi-threaded access to a results instance is not supported, for + performance reasons, so such Element Data is not safe to use from several + threads at once. It takes no lock to make those reads safe. The caller is + responsible for thread-safe access, and either gives each thread its own + Flow Data or coordinates its own access. """ diff --git a/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py b/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py index 4b00ddb1..96ccbfb5 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py @@ -1,11 +1,11 @@ """ -The documentation of Element Data says it is safe to read from several threads -once the Flow Element that writes it has finished, and is not catered for when -written from several threads or read while another thread writes. It also says -that Element Data reading from a results instance, which holds unmanaged -memory, is not safe to use from several threads at once and that the caller is -responsible, as the thread safety page of the specification requires. +The documentation of Element Data says it is written only by the Flow Element +that provides it, or those that share it, and only while they process, and +that it is then safe to read from several threads. It also says that Element +Data reading from a results instance, which references unmanaged memory, is +not safe to use from several threads at once, takes no lock and leaves the +caller responsible, as the thread safety page of the specification requires. """ import unittest @@ -15,13 +15,13 @@ class ElementDataDocumentationTests(unittest.TestCase): - def test_the_documentation_says_what_is_safe_with_threads(self): + def test_the_documentation_says_who_writes_and_when(self): text = " ".join(ElementData.__doc__.split()) + self.assertIn( + "written only by the Flow Element that provides it, or by the " + "Flow Elements that share it, and only while they process", text) self.assertIn("safe to read from several threads", text) - self.assertIn("once the Flow Element that writes it has finished", text) - self.assertIn("not catered for", text) - self.assertIn("written from several threads", text) - self.assertIn("read while another thread writes", text) + self.assertIn("used by one thread at a time", text) def test_the_documentation_says_results_instances_are_the_exception(self): text = " ".join(ElementData.__doc__.split()) @@ -30,6 +30,7 @@ def test_the_documentation_says_results_instances_are_the_exception(self): self.assertIn( "Multi-threaded access to a results instance is not supported, " "for performance reasons", text) + self.assertIn("takes no lock", text) self.assertIn("caller is responsible for thread-safe access", text) From def9267a9d001e1fa48b4f25e76989f77e55f8e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eddie=20Vereb=C3=A9lyi?= Date: Wed, 18 Nov 2026 22:53:55 +0000 Subject: [PATCH 5/5] Say the no lock test looks only at the instance test_an_instance_holds_no_lock in both on-premise packages looks only at the attributes of one SwigData instance. A lock at module level, on the engine or in AspectData would still pass it. Its docstring now says it is a smoke test, so no one reads more into it than that. The rule it checks is in the results instance section of the thread safety page, as merged in specifications 52e9b047. --- .../tests/test_swig_data_threads.py | 3 +++ .../tests/test_swig_data_threads.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py index d0fa5c02..538cc0b7 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py @@ -56,6 +56,9 @@ def test_the_documentation_says_the_caller_coordinates_threads(self): self.assertIn("caller is responsible for thread-safe access", text) def test_an_instance_holds_no_lock(self): + """A smoke test. It looks only at the instance's own attributes, + so a lock at module level, on the engine or in AspectData would + still pass.""" data = SwigData(FakeElement(), object()) locks = [name for name, value in vars(data).items() if isinstance(value, LOCK_TYPES)] diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py index cbbf193f..41313401 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py @@ -56,6 +56,9 @@ def test_the_documentation_says_the_caller_coordinates_threads(self): self.assertIn("caller is responsible for thread-safe access", text) def test_an_instance_holds_no_lock(self): + """A smoke test. It looks only at the instance's own attributes, + so a lock at module level, on the engine or in AspectData would + still pass.""" data = SwigData(FakeElement(), object()) locks = [name for name, value in vars(data).items() if isinstance(value, LOCK_TYPES)]