From ef2ce6483ae716c1a1ac78e1e9319c7bf68ae509 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 22:16:31 +0000 Subject: [PATCH 01/11] Test that a finished Flow Data and finished Element Data refuse writes Each package that defines a Flow Element gains a test that processes every Flow Element the package and its examples define, then writes to its Element Data and expects an error. It also checks that it covers every Flow Element class the package's own modules define. The pipeline core test also checks that a finished Flow Data refuses Evidence, Element Data and errors, that a second process raises even when process exceptions are suppressed, and that Flow Elements sharing one Element Data write it while each processes. --- .../tests/test_finished_writes.py | 100 ++++++++ .../tests/test_finished_writes.py | 81 +++++++ .../tests/test_finished_writes.py | 102 ++++++++ .../tests/test_finished_writes.py | 81 +++++++ .../tests/test_finished_writes.py | 89 +++++++ location/tests/test_finished_writes.py | 97 ++++++++ .../tests/test_finished_writes.py | 135 +++++++++++ .../tests/test_finished_writes.py | 224 ++++++++++++++++++ .../tests/test_finished_writes.py | 92 +++++++ .../tests/test_finished_writes.py | 85 +++++++ .../tests/test_finished_writes.py | 80 +++++++ .../tests/test_finished_writes.py | 89 +++++++ 12 files changed, 1255 insertions(+) create mode 100644 device-detection/fiftyone_devicedetection_cloud/tests/test_finished_writes.py create mode 100644 device-detection/fiftyone_devicedetection_onpremise/tests/test_finished_writes.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_cloud/tests/test_finished_writes.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_finished_writes.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_translation/tests/test_finished_writes.py create mode 100644 location/tests/test_finished_writes.py create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_finished_writes.py create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/tests/test_finished_writes.py create mode 100644 pipeline/fiftyone_pipeline_engines/tests/test_finished_writes.py create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_finished_writes.py create mode 100644 pipeline/fiftyone_pipeline_translation/tests/test_finished_writes.py diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_finished_writes.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_finished_writes.py new file mode 100644 index 00000000..fd554cda --- /dev/null +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_finished_writes.py @@ -0,0 +1,100 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. +""" + +import importlib +import inspect +import json +import pkgutil +import unittest + +import fiftyone_devicedetection_cloud +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_devicedetection_cloud.devicedetection_cloud import \ + DeviceDetectionCloud +from fiftyone_devicedetection_cloud.hardwareprofile_cloud import \ + HardwareProfileCloud + +FINISHED = "finished processing" + +# FlowElements that only other FlowElements build on. +BASES = set() + + +class StandInRequestEngine(FlowElement): + """Stands in for a Cloud Request Engine whose answer has an empty + section for each of the given data keys.""" + + def __init__(self, keys): + super(StandInRequestEngine, self).__init__() + self.datakey = "cloud" + self.flow_element_properties = {key: {} for key in keys} + self.answer = json.dumps({key: {} for key in keys}) + + def process_internal(self, flowdata): + flowdata.set_element_data( + ElementDataDictionary(self, {"cloud": self.answer})) + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + processed = set() + for element in [DeviceDetectionCloud(), HardwareProfileCloud()]: + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + flowdata = builder \ + .add(StandInRequestEngine([element.datakey])) \ + .add(element).build().create_flowdata() + flowdata.process() + data = flowdata.get(element.datakey) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + processed.add(type(element)) + self.assertEqual( + defined_flow_elements(fiftyone_devicedetection_cloud) - BASES, processed) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_finished_writes.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_finished_writes.py new file mode 100644 index 00000000..bcad8a34 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_finished_writes.py @@ -0,0 +1,81 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. Reading values from the results is not a write. +""" + +import importlib +import inspect +import pkgutil +import unittest + +import fiftyone_devicedetection_onpremise +from fiftyone_devicedetection_onpremise.devicedetection_onpremise import \ + DeviceDetectionOnPremise +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +DATA_FILE = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash" +USER_AGENT = ("Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) " + "AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148") +FINISHED = "finished processing" + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + engine = DeviceDetectionOnPremise( + data_file_path=DATA_FILE, licence_keys="", auto_update=False) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("header.user-agent", USER_AGENT) + flowdata.process() + data = flowdata.get("device") + self.assertTrue(data.ismobile.value()) + with self.assertRaises(Exception) as caught: + data.ismobile = False + self.assertIn(FINISHED, str(caught.exception)) + self.assertTrue(data.ismobile.value()) + self.assertEqual( + {DeviceDetectionOnPremise}, + defined_flow_elements(fiftyone_devicedetection_onpremise)) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_cloud/tests/test_finished_writes.py b/ip-intelligence/fiftyone_ipintelligence_cloud/tests/test_finished_writes.py new file mode 100644 index 00000000..150c5726 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_cloud/tests/test_finished_writes.py @@ -0,0 +1,102 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. +""" + +import importlib +import inspect +import json +import pkgutil +import unittest + +import fiftyone_ipintelligence_cloud +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_ipintelligence_cloud.countries_translation_cloud import \ + CountriesTranslationCloud +from fiftyone_ipintelligence_cloud.ipintelligence_cloud import \ + IpIntelligenceCloud +from fiftyone_ipintelligence_cloud.weighted_cloud_engine import \ + WeightedCloudEngine + +FINISHED = "finished processing" + +# FlowElements that only other FlowElements build on. +BASES = {WeightedCloudEngine} + + +class StandInRequestEngine(FlowElement): + """Stands in for a Cloud Request Engine whose answer has an empty + section for each of the given data keys.""" + + def __init__(self, keys): + super(StandInRequestEngine, self).__init__() + self.datakey = "cloud" + self.flow_element_properties = {key: {} for key in keys} + self.answer = json.dumps({key: {} for key in keys}) + + def process_internal(self, flowdata): + flowdata.set_element_data( + ElementDataDictionary(self, {"cloud": self.answer})) + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + processed = set() + for element in [IpIntelligenceCloud(), CountriesTranslationCloud()]: + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + flowdata = builder \ + .add(StandInRequestEngine([element.datakey])) \ + .add(element).build().create_flowdata() + flowdata.process() + data = flowdata.get(element.datakey) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + processed.add(type(element)) + self.assertEqual( + defined_flow_elements(fiftyone_ipintelligence_cloud) - BASES, processed) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_finished_writes.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_finished_writes.py new file mode 100644 index 00000000..251e8b9e --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_finished_writes.py @@ -0,0 +1,81 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. Reading values from the results is not a write. +""" + +import importlib +import inspect +import pkgutil +import unittest + +import fiftyone_ipintelligence_onpremise +from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise import \ + IpIntelligenceOnPremise +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +from .constants import TestConstants + +DATA_FILE = TestConstants.get_data_file_path() +FINISHED = "finished processing" + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + engine = IpIntelligenceOnPremise( + data_file_path=DATA_FILE, licence_keys="", auto_update=False) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("query.client-ip", "185.28.167.77") + flowdata.process() + data = flowdata.get("ip") + name = data.get("registeredname") + with self.assertRaises(Exception) as caught: + data.registeredname = "written after processing" + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual(name.value(), data.get("registeredname").value()) + self.assertEqual( + {IpIntelligenceOnPremise}, + defined_flow_elements(fiftyone_ipintelligence_onpremise)) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_finished_writes.py b/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_finished_writes.py new file mode 100644 index 00000000..3025c09e --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_finished_writes.py @@ -0,0 +1,89 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. +""" + +import importlib +import inspect +import pkgutil +import unittest + +import fiftyone_ipintelligence_translation +from fiftyone_ipintelligence_translation.countries_translation_engine import \ + CountriesTranslationEngine +from fiftyone_ipintelligence_translation.countries_translation_engine_builder \ + import CountriesTranslationEngineBuilder +from fiftyone_ipintelligence_translation.country_code_translation_engine \ + import CountryCodeTranslationEngine +from fiftyone_ipintelligence_translation \ + .country_code_translation_engine_builder import \ + CountryCodeTranslationEngineBuilder +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +from .mock_ip_engine import mock_ip_element + +FINISHED = "finished processing" + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + engines = [CountryCodeTranslationEngineBuilder().build(), + CountriesTranslationEngineBuilder().build()] + builder = PipelineBuilder().add(mock_ip_element(["DE"], ["DE"])) + for engine in engines: + builder.add(engine) + flowdata = builder.build().create_flowdata() + flowdata.process() + for engine in engines: + data = flowdata.get(engine.datakey) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual( + {type(engine) for engine in engines}, + defined_flow_elements(fiftyone_ipintelligence_translation)) + self.assertEqual( + {CountriesTranslationEngine, CountryCodeTranslationEngine}, + {type(engine) for engine in engines}) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/location/tests/test_finished_writes.py b/location/tests/test_finished_writes.py new file mode 100644 index 00000000..bfb8f7a9 --- /dev/null +++ b/location/tests/test_finished_writes.py @@ -0,0 +1,97 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. +""" + +import importlib +import inspect +import json +import pkgutil +import unittest + +import fiftyone_location +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_location.location_cloud import LocationCloud + +FINISHED = "finished processing" + +# FlowElements that only other FlowElements build on. +BASES = set() + + +class StandInRequestEngine(FlowElement): + """Stands in for a Cloud Request Engine whose answer has an empty + section for each of the given data keys.""" + + def __init__(self, keys): + super(StandInRequestEngine, self).__init__() + self.datakey = "cloud" + self.flow_element_properties = {key: {} for key in keys} + self.answer = json.dumps({key: {} for key in keys}) + + def process_internal(self, flowdata): + flowdata.set_element_data( + ElementDataDictionary(self, {"cloud": self.answer})) + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + processed = set() + for element in [LocationCloud()]: + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + flowdata = builder \ + .add(StandInRequestEngine([element.datakey])) \ + .add(element).build().create_flowdata() + flowdata.process() + data = flowdata.get(element.datakey) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + processed.add(type(element)) + self.assertEqual( + defined_flow_elements(fiftyone_location) - BASES, processed) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_finished_writes.py new file mode 100644 index 00000000..520caa6b --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_finished_writes.py @@ -0,0 +1,135 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines, and every one its examples define, is +processed, then a write to its ElementData is expected to fail, as the thread +safety page of the specification requires. +""" + +import importlib +import importlib.util +import inspect +import json +import os +import pkgutil + +import fiftyone_pipeline_cloudrequestengine +from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine +from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import \ + CloudRequestEngine +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +from .classes.cloudrequestengine_testbase import CloudRequestEngineTestsBase + +FINISHED = "finished processing" +EXAMPLES = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "examples", "src", "fiftyone_pipeline_examples", "cloud") + + +class StandInRequestEngine(FlowElement): + """Stands in for a Cloud Request Engine whose answer has an empty + section for each of the given data keys.""" + + def __init__(self, keys): + super(StandInRequestEngine, self).__init__() + self.datakey = "cloud" + self.flow_element_properties = {key: {} for key in keys} + self.answer = json.dumps({key: {} for key in keys}) + + def process_internal(self, flowdata): + flowdata.set_element_data( + ElementDataDictionary(self, {"cloud": self.answer})) + + +def load_example(name): + """An example module, loaded from its file under its own name.""" + spec = importlib.util.spec_from_file_location( + "example_" + name, os.path.join(EXAMPLES, name + ".py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(CloudRequestEngineTestsBase): + + def assert_writes_fail(self, flowdata, key): + data = flowdata.get(key) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + + def test_every_flow_element_refuses_writes_once_finished(self): + cloud = CloudRequestEngine({ + "license_key": "a-licence-key", + "requested_properties": ["device.ismobile"], + "http_client": self.mock_http(), + }) + engine = CloudEngine() + engine.datakey = "device" + flowdata = PipelineBuilder().add(cloud).add(engine).build() \ + .create_flowdata() + flowdata.evidence.add("query.User-Agent", "iPhone") + flowdata.process() + self.assert_writes_fail(flowdata, "cloud") + self.assert_writes_fail(flowdata, "device") + self.assertEqual( + {CloudEngine, CloudRequestEngine}, + defined_flow_elements(fiftyone_pipeline_cloudrequestengine)) + + def test_every_example_flow_element_refuses_writes_once_finished(self): + engines = load_example("engines") + simple = load_example("simple_cloud_engine") + elements = [engines.DeviceDetectionCloud(), + engines.IpIntelligenceCloud(), + engines.CountriesTranslationCloud(), + simple.SimpleCloudEngine()] + for element in elements: + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + flowdata = builder \ + .add(StandInRequestEngine([element.datakey])) \ + .add(element).build().create_flowdata() + flowdata.process() + self.assert_writes_fail(flowdata, element.datakey) + + +if __name__ == "__main__": + import unittest + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py new file mode 100644 index 00000000..c58fdd2c --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py @@ -0,0 +1,224 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +A FlowData that has finished processing refuses Evidence, ElementData and +errors, and ElementData refuses a write once the FlowElement that writes it +has finished, as the thread safety page of the specification requires. The +last tests process every FlowElement this package defines, and its example +FlowElements, then write to each one's ElementData and expect the error. +""" + +import importlib +import importlib.util +import inspect +import os +import pkgutil +import unittest + +import fiftyone_pipeline_core +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.flowerror import FlowError +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +FINISHED = "finished processing" + +# FlowElements that only other FlowElements build on, which have no +# ElementData of their own to write. +BASES = {FlowElement} + +EXAMPLES = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "examples") + + +class WritingElement(FlowElement): + """Adds data and then writes it, as a FlowElement may while it + processes.""" + + def __init__(self, datakey="writer", value="first"): + super(WritingElement, self).__init__() + self.datakey = datakey + self.value = value + + def process_internal(self, flowdata): + data = flowdata.data.get(self.datakey) + if data is None: + data = ElementDataDictionary(self, {}) + flowdata.set_element_data(data) + data.written = self.value + + +def process(*elements, **settings): + builder = PipelineBuilder(dict({"add_javascript_builder": False, + "use_setheader_properties": False}, + **settings)) + for element in elements: + builder.add(element) + flowdata = builder.build().create_flowdata() + flowdata.process() + return flowdata + + +def load_example(name): + """An example module, loaded from its file under its own name.""" + spec = importlib.util.spec_from_file_location( + "example_" + name, os.path.join(EXAMPLES, name + ".py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedFlowDataTests(unittest.TestCase): + + def test_evidence_cannot_be_added_once_processed(self): + flowdata = process(WritingElement()) + with self.assertRaises(Exception) as caught: + flowdata.evidence.add("query.later", "value") + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + flowdata.evidence.add_from_dict({"query.later": "value"}) + self.assertIn(FINISHED, str(caught.exception)) + self.assertIsNone(flowdata.evidence.get("query.later")) + + def test_element_data_cannot_be_added_once_processed(self): + element = WritingElement("late") + flowdata = process(WritingElement()) + with self.assertRaises(Exception) as caught: + flowdata.set_element_data(ElementDataDictionary(element, {})) + self.assertIn(FINISHED, str(caught.exception)) + self.assertNotIn("late", flowdata.data) + + def test_errors_cannot_be_added_once_processed(self): + flowdata = process(WritingElement()) + with self.assertRaises(Exception) as caught: + flowdata.set_error(FlowError("late", Exception("late"), "")) + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual({}, flowdata.errors) + + def test_a_second_process_raises_even_when_exceptions_are_suppressed(self): + flowdata = process(WritingElement(), + suppress_process_exceptions=True) + with self.assertRaises(Exception) as caught: + flowdata.process() + self.assertIn("already processed", str(caught.exception)) + self.assertEqual({}, flowdata.errors) + + def test_evidence_can_be_added_before_processing(self): + flowdata = PipelineBuilder().add(WritingElement()).build() \ + .create_flowdata() + flowdata.evidence.add("query.early", "value") + self.assertEqual("value", flowdata.evidence.get("query.early")) + + +class FinishedElementDataTests(unittest.TestCase): + + def test_data_written_while_processing_cannot_be_written_after(self): + flowdata = process(WritingElement()) + data = flowdata.get("writer") + self.assertEqual("first", data.written) + with self.assertRaises(Exception) as caught: + data.written = "after" + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual("first", data.written) + + def test_an_element_sharing_the_data_writes_it_while_it_processes(self): + flowdata = process(WritingElement("shared", "first"), + WritingElement("shared", "second")) + data = flowdata.get("shared") + self.assertEqual("second", data.written) + with self.assertRaises(Exception): + data.written = "after" + + def test_the_data_keeps_its_own_state_once_finished(self): + data = process(WritingElement()).get("writer") + data._kept = "a value kept once read" + self.assertEqual("a value kept once read", data._kept) + + def test_closing_the_flow_data_is_not_a_write(self): + class ClosingData(ElementDataDictionary): + def close(self): + self.closed = True + + class ClosingElement(FlowElement): + def __init__(self): + super(ClosingElement, self).__init__() + self.datakey = "closing" + + def process_internal(self, flowdata): + flowdata.set_element_data(ClosingData(self, {})) + + flowdata = process(ClosingElement()) + flowdata.close() + self.assertTrue(flowdata.get("closing").closed) + + +class EveryFlowElementTests(unittest.TestCase): + + def assert_writes_fail(self, flowdata, key): + data = flowdata.get(key) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + + def test_the_package_flow_elements_refuse_writes_once_finished(self): + flowdata = PipelineBuilder().build().create_flowdata() + flowdata.evidence.add("header.user-agent", "a user agent") + flowdata.process() + processed = set() + for element in flowdata.pipeline.flow_elements: + self.assert_writes_fail(flowdata, element.datakey) + processed.add(type(element)) + self.assertEqual( + defined_flow_elements(fiftyone_pipeline_core) - BASES, processed) + + def test_the_example_flow_elements_refuse_writes_once_finished(self): + for name, element in ( + ("simple_flow_element", "SimpleFlowElement"), + ("client_side_evidence_custom_flow_element", + "AstrologyFlowElement")): + module = load_example(name) + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + flowdata = builder.add(getattr(module, element)()).build() \ + .create_flowdata() + flowdata.evidence.add("query.dateOfBirth", "1992-12-18") + flowdata.process() + self.assert_writes_fail(flowdata, "astrology") + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_derivedproperty/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_finished_writes.py new file mode 100644 index 00000000..e2754fcd --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_finished_writes.py @@ -0,0 +1,92 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines, and the one its example defines, is +processed, then a write to its ElementData is expected to fail, as the +thread safety page of the specification requires. The derived property +elements of a pipeline share one ElementData, which each writes while it +processes and none writes once they have all finished. +""" + +import importlib +import importlib.util +import inspect +import os +import pkgutil + +import pytest + +import fiftyone_pipeline_derivedproperty +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_derivedproperty import DerivedPropertyElementBuilder +from fiftyone_pipeline_derivedproperty.derived_property_element import \ + DerivedPropertyElement + +from .helpers import SourceElement, band_script, pipeline_of, processed + +FINISHED = "finished processing" +EXAMPLES = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "examples") + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +def derived(name): + return (DerivedPropertyElementBuilder() + .add_script(name, band_script(name, "src.N")).build()) + + +def test_every_derived_element_writes_only_while_it_processes(): + flowdata = processed(pipeline_of( + [SourceElement("src", {"N": 12}), derived("Demo"), derived("Other")])) + data = flowdata.derived + assert data.get("Demo").value() == "High" + assert data.get("Other").value() == "High" + with pytest.raises(Exception, match=FINISHED): + data.set_value("Demo", "written after processing") + with pytest.raises(Exception, match=FINISHED): + data.written_after_processing = True + assert data.get("Demo").value() == "High" + assert defined_flow_elements(fiftyone_pipeline_derivedproperty) == \ + {DerivedPropertyElement} + + +def test_the_example_flow_element_refuses_writes_once_finished(): + spec = importlib.util.spec_from_file_location( + "example_screen_element", os.path.join(EXAMPLES, "screen_element.py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + flowdata = processed(pipeline_of([module.ScreenElement()])) + with pytest.raises(Exception, match=FINISHED): + flowdata.get("screen").written_after_processing = True \ No newline at end of file diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_engines/tests/test_finished_writes.py new file mode 100644 index 00000000..50640c72 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines/tests/test_finished_writes.py @@ -0,0 +1,85 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +The engines package defines only the Engine that other FlowElements build +on. Its example engine is processed, then a write to its ElementData is +expected to fail, as the thread safety page of the specification requires. +""" + +import importlib +import importlib.util +import inspect +import os +import pkgutil +import unittest + +import fiftyone_pipeline_engines +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_engines.engine import Engine + +FINISHED = "finished processing" +EXAMPLES = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "examples") + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_the_package_defines_only_the_engine_others_build_on(self): + self.assertEqual( + {Engine}, defined_flow_elements(fiftyone_pipeline_engines)) + + def test_the_example_engine_refuses_writes_once_finished(self): + spec = importlib.util.spec_from_file_location( + "example_simple_onpremise_engine", + os.path.join(EXAMPLES, "simple_onpremise_engine.py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + engine = module.SimpleOnPremiseEngine(module.DATA_FILE) + pipeline = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(engine).build() + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + flowdata.evidence.add(module.DATE_OF_BIRTH, "1992-12-18") + flowdata.process() + with self.assertRaises(Exception) as caught: + flowdata.get("astrology").written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_finished_writes.py new file mode 100644 index 00000000..6c4b6400 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_finished_writes.py @@ -0,0 +1,80 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, as the thread safety +page of the specification requires. The usage sharing element adds no +ElementData, so a finished FlowData holds none for it, and the FlowData +refuses one being added. +""" + +import importlib +import inspect +import pkgutil +import unittest + +import fiftyone_pipeline_engines_fiftyone +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_engines_fiftyone.share_usage import ShareUsage + +FINISHED = "finished processing" + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + # Nothing listens on the discard port, and one record is far short + # of a package, so nothing is sent. + usage = ShareUsage(requested_package_size=1000, + endpoint="http://127.0.0.1:9") + pipeline = PipelineBuilder().add(usage).build() + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "a user agent") + flowdata.process() + self.assertNotIn(usage.datakey, flowdata.data) + with self.assertRaises(Exception) as caught: + flowdata.set_element_data(ElementDataDictionary(usage, {})) + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual( + {ShareUsage}, + defined_flow_elements(fiftyone_pipeline_engines_fiftyone)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_translation/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_translation/tests/test_finished_writes.py new file mode 100644 index 00000000..b0d5674b --- /dev/null +++ b/pipeline/fiftyone_pipeline_translation/tests/test_finished_writes.py @@ -0,0 +1,89 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. +""" + +import importlib +import inspect +import pkgutil +import unittest + +import fiftyone_pipeline_translation +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_translation.translation_engine_base import ( + TranslationEngineBase) + +FINISHED = "finished processing" + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class SourceElement(FlowElement): + """Gives fixed names to translate.""" + + def __init__(self): + super(SourceElement, self).__init__() + self.datakey = "source" + + def process_internal(self, flowdata): + flowdata.set_element_data( + ElementDataDictionary(self, {"names": ["Germany"]})) + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + engine = TranslationEngineBase( + "source", [("Names", "NamesTranslated")], + {"countries.fr_FR.yml": "Germany: Allemagne\n"}, "fr_FR") + flowdata = PipelineBuilder().add(SourceElement()).add(engine) \ + .build().create_flowdata() + flowdata.process() + data = flowdata.get(engine.datakey) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual( + {TranslationEngineBase}, + defined_flow_elements(fiftyone_pipeline_translation)) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From eae137e2390f9cad8ad73713cda131d51f59ea87 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 22:16:32 +0000 Subject: [PATCH 02/11] Refuse writes to a finished Flow Data and to finished Element Data FlowData raises once processing has finished when Evidence, Element Data or an error is added, and a second process raises without adding an error. ElementData records when the Flow Element that writes it has finished, and then setting a public attribute, or a write method such as the derived property data's set_value, raises. A Flow Element that shares the instance opens it again while it processes. Names that start with an underscore hold the data's own state, such as values kept once read, and closing the Flow Data is not a write. The checks read a field set when processing ends and take no lock. --- .../src/fiftyone_pipeline_core/elementdata.py | 41 +++++++++ .../src/fiftyone_pipeline_core/evidence.py | 10 +++ .../src/fiftyone_pipeline_core/flowdata.py | 88 +++++++++++++------ .../src/fiftyone_pipeline_core/messages.py | 15 +++- .../derived_property_data.py | 8 +- 5 files changed, 132 insertions(+), 30 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py index 5ee546d3..19d42eaf 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py @@ -20,6 +20,8 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from .messages import Messages + class ElementData(object): @@ -49,6 +51,45 @@ def __init__(self, flow_element): self.flow_element = flow_element + # Set once the FlowElement that writes the data has finished processing, + # and cleared while one that shares the instance processes. A class + # default, so data whose constructor does not call this one has it too. + _finished = False + + def __setattr__(self, name, value): + + """! + Set an attribute, unless the FlowElement that writes the data has + finished processing, when a public attribute cannot be set. Names + starting with an underscore hold the data's own state, such as + values kept once read, and are not writes. + + @raise Exception: when the FlowElement has finished processing + """ + + if self._finished and not name.startswith("_"): + self._raise_finished(name) + object.__setattr__(self, name, value) + + def _check_writable(self, name): + + """! + Raise unless the FlowElement that writes the data is processing. A + write method calls this first. + + @type name: str + @param name: what is being written, for the message + @raise Exception: when the FlowElement has finished processing + """ + + if self._finished: + self._raise_finished(name) + + def _raise_finished(self, name): + flow_element = self.__dict__.get("flow_element") + key = getattr(flow_element, "datakey", "") + raise Exception(Messages.ELEMENT_DATA_FINISHED.format(name, key)) + def get(self, key): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py index ee6b8f5d..4f59b61d 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py @@ -20,6 +20,8 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from .messages import Messages + # The key the web integration gives the web request itself, which the usage # sharing element reads. No element uses it as evidence, so filter() always # sets it aside, whatever the elements' filters accept. @@ -51,8 +53,12 @@ def add(self, key, value): @type value: mixed @param value: a piece of evidence + @raise Exception: once the FlowData has finished processing """ + if self.flowdata.processed: + raise Exception(Messages.FLOW_DATA_FINISHED.format("Evidence")) + folded = str(key).lower() if folded in self.keys: key = self.keys[folded] @@ -66,8 +72,12 @@ def add_from_dict(self, evidence_dictionary): @type evidence_dictionary: dict @param evidence_dictionary: Dict of evidence + @raise Exception: once the FlowData has finished processing """ + if self.flowdata.processed: + raise Exception(Messages.FLOW_DATA_FINISHED.format("Evidence")) + if not type(evidence_dictionary) is dict: self.flowdata.set_error("core", "Must pass valid dictionary.") diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py index 4f33afee..caab8507 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py @@ -20,12 +20,22 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from .elementdata import ElementData from .evidence import Evidence from .flowerror import FlowError -from .messages import Messages +from .messages import Messages import traceback import sys + +def _set_finished(element_data, finished): + + # Records whether the FlowElement that writes the data has finished, which + # ElementData checks before a write. Data of another type is left alone. + if isinstance(element_data, ElementData): + object.__setattr__(element_data, "_finished", finished) + + class FlowData: """! FlowData is created by a specific Pipeline @@ -60,37 +70,44 @@ def process(self): @rtype: FlowData @return: Returns flowdata - @raise Exception: the first error a flow element raised, or the - error saying this FlowData has already been processed, unless the - Pipeline suppresses process exceptions. Every error is in errors - either way. + @raise Exception: the first error a flow element raised, unless the + Pipeline suppresses process exceptions, and every error is in errors + either way. A second call raises the error saying this FlowData has + already been processed, as a finished FlowData takes no more errors. """ - if not self.processed: - # filter evidence before processing - try: - self.evidence.filter() - except Exception as e: - flow_error = FlowError("global", e, traceback.format_exc()) - self.set_error(flow_error) - - for flow_element in self.pipeline.flow_elements: - if self.stopped is not True: - # All errors are caught and stored in an errors array keyed by the - # flowElement that set the error - try: - flow_element.process(self) - except Exception: - flow_error = FlowError(flow_element.datakey, sys.exc_info()[1], traceback.format_exc()) - self.set_error(flow_error) - - # Set processed flag to true. flowdata can only be processed once - self.processed = True - else: - flow_error = FlowError("global", Exception(Messages.FLOW_DATA_PROCESSED), Messages.FLOW_DATA_PROCESSED) + # A finished FlowData takes no more errors, so a second call raises. + if self.processed: + raise Exception(Messages.FLOW_DATA_PROCESSED) + + # filter evidence before processing + try: + self.evidence.filter() + except Exception as e: + flow_error = FlowError("global", e, traceback.format_exc()) self.set_error(flow_error) + for flow_element in self.pipeline.flow_elements: + if self.stopped is not True: + key = str(flow_element.datakey).lower() + # A FlowElement that shares data another added may write it + # again while it processes. + _set_finished(self.data.get(key), False) + # All errors are caught and stored in an errors array keyed by the + # flowElement that set the error + try: + flow_element.process(self) + except Exception: + flow_error = FlowError(flow_element.datakey, sys.exc_info()[1], traceback.format_exc()) + self.set_error(flow_error) + finally: + _set_finished(self.data.get(key), True) + + # Set processed flag to true. flowdata can only be processed once, + # and is read only from now on. + self.processed = True + if self.errors and self.pipeline.suppress_process_exceptions is False: errored_flow_element_key = next(iter(self.errors)) flowErrorObj = self.errors[errored_flow_element_key] @@ -118,7 +135,14 @@ def close(self): # attribute it does not have as the name of a property. close = getattr(type(element_data), "close", None) if callable(close): - close(element_data) + # Releasing what the data holds is not a write to it. + finished = (isinstance(element_data, ElementData) + and element_data._finished) + _set_finished(element_data, False) + try: + close(element_data) + finally: + _set_finished(element_data, finished) def __enter__(self): return self @@ -210,9 +234,13 @@ def set_element_data(self, element_data): @type element_data: ElementData @param element_data: elementData to be added to flowdata + @raise Exception: once this FlowData has finished processing """ + if self.processed: + raise Exception(Messages.FLOW_DATA_FINISHED.format("ElementData")) + # Kept under the data key in lower case, which is how get reads it. self.data[element_data.flow_element.datakey.lower()] = element_data @@ -224,9 +252,13 @@ def set_error(self, flow_error): @type error: FlowError @param error: Flow Error + @raise Exception: once this FlowData has finished processing """ + if self.processed: + raise Exception(Messages.FLOW_DATA_FINISHED.format("an error")) + self.errors[flow_error.flow_element] = flow_error log_message = "Error occurred during processing" diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py index 05f2996e..913cad1c 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py @@ -46,4 +46,17 @@ class Messages(): "Property '{}' is not present in the FlowData against '{}' ElementData. " # FlowData already processed. - FLOW_DATA_PROCESSED = "FlowData already processed" \ No newline at end of file + FLOW_DATA_PROCESSED = "FlowData already processed" + + # A write to a FlowData that has finished processing. This takes what was + # being added as a format argument. + FLOW_DATA_FINISHED = ( + "This FlowData has finished processing, so {} cannot be added to it. " + "A FlowData is read only once processing has finished.") + + # A write to ElementData once its FlowElement has finished. This takes + # the name written and the element's data key as format arguments. + ELEMENT_DATA_FINISHED = ( + "'{}' cannot be written to the '{}' ElementData, because the " + "FlowElement that writes it has finished processing. ElementData is " + "written only while the FlowElement that provides it processes.") \ No newline at end of file diff --git a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_data.py b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_data.py index dcca50e6..96e9c9b7 100644 --- a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_data.py +++ b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_data.py @@ -36,5 +36,11 @@ def __init__(self, flow_element): super().__init__(flow_element, {}) def set_value(self, name, value): - """! Writes the value of one property. """ + """! + Writes the value of one property. + + @raise Exception: once every element that writes the data has + finished processing + """ + self._check_writable(name) self.contents[name.lower()] = value From 91fc9335e2c91a61f97d31840ddb017c18d5ff71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eddie=20Vereb=C3=A9lyi?= Date: Wed, 18 Nov 2026 22:04:07 +0000 Subject: [PATCH 03/11] Test that finished Element Data refuses writes to its dictionary The contents of ElementDataDictionary, AspectDataDictionary and CloudData, and a hardware profile, which is itself a dict, can still be written once the Flow Element that provides them has finished. Pipeline core now checks each dict write method and that the contents stay a dict. Each package's check over every Flow Element also writes an item, and the Device Detection cloud check writes to a hardware profile in each way. --- .../tests/test_finished_writes.py | 40 +++++++++++ .../tests/test_finished_writes.py | 11 +++ .../tests/test_finished_writes.py | 10 +++ location/tests/test_finished_writes.py | 11 +++ .../tests/test_finished_writes.py | 12 ++++ .../tests/test_finished_writes.py | 72 +++++++++++++++++++ .../tests/test_finished_writes.py | 15 +++- .../tests/test_finished_writes.py | 12 ++++ .../tests/test_finished_writes.py | 10 +++ 9 files changed, 192 insertions(+), 1 deletion(-) diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_finished_writes.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_finished_writes.py index fd554cda..4d0c9e49 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/test_finished_writes.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_finished_writes.py @@ -44,6 +44,14 @@ FINISHED = "finished processing" + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + # FlowElements that only other FlowElements build on. BASES = set() @@ -91,10 +99,42 @@ def test_every_flow_element_refuses_writes_once_finished(self): with self.assertRaises(Exception) as caught: data.written_after_processing = True self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) processed.add(type(element)) self.assertEqual( defined_flow_elements(fiftyone_devicedetection_cloud) - BASES, processed) + def test_a_hardware_profile_refuses_writes_once_finished(self): + writes = ( + ("attribute", lambda p: setattr(p, "written", True)), + ("[]=", lambda p: p.__setitem__("written", True)), + ("del", lambda p: p.__delitem__("hardwarevendor")), + ("update", lambda p: p.update(written=True)), + ("pop", lambda p: p.pop("hardwarevendor")), + ("popitem", lambda p: p.popitem()), + ("setdefault", lambda p: p.setdefault("written", True)), + ("clear", lambda p: p.clear()), + ("|=", lambda p: p.__ior__({"written": True})), + ) + element = HardwareProfileCloud() + request = StandInRequestEngine([element.datakey]) + request.answer = json.dumps({element.datakey: { + "profiles": [{"hardwarevendor": "Apple"}]}}) + flowdata = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(request).add(element).build().create_flowdata() + flowdata.process() + profile = flowdata.get(element.datakey).get("profiles")[0] + for name, write in writes: + with self.subTest(name): + with self.assertRaises(Exception) as caught: + write(profile) + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual(["hardwarevendor"], list(profile)) + self.assertEqual("Apple", profile.hardwarevendor.value()) + if __name__ == "__main__": unittest.main() \ No newline at end of file diff --git a/ip-intelligence/fiftyone_ipintelligence_cloud/tests/test_finished_writes.py b/ip-intelligence/fiftyone_ipintelligence_cloud/tests/test_finished_writes.py index 150c5726..4ab707da 100644 --- a/ip-intelligence/fiftyone_ipintelligence_cloud/tests/test_finished_writes.py +++ b/ip-intelligence/fiftyone_ipintelligence_cloud/tests/test_finished_writes.py @@ -46,6 +46,14 @@ FINISHED = "finished processing" + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + # FlowElements that only other FlowElements build on. BASES = {WeightedCloudEngine} @@ -93,6 +101,9 @@ def test_every_flow_element_refuses_writes_once_finished(self): with self.assertRaises(Exception) as caught: data.written_after_processing = True self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) processed.add(type(element)) self.assertEqual( defined_flow_elements(fiftyone_ipintelligence_cloud) - BASES, processed) diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_finished_writes.py b/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_finished_writes.py index 3025c09e..26e75e85 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_finished_writes.py +++ b/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_finished_writes.py @@ -49,6 +49,13 @@ FINISHED = "finished processing" +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + def defined_flow_elements(package): """Every FlowElement class defined in the package's own modules.""" found = set() @@ -77,6 +84,9 @@ def test_every_flow_element_refuses_writes_once_finished(self): with self.assertRaises(Exception) as caught: data.written_after_processing = True self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) self.assertEqual( {type(engine) for engine in engines}, defined_flow_elements(fiftyone_ipintelligence_translation)) diff --git a/location/tests/test_finished_writes.py b/location/tests/test_finished_writes.py index bfb8f7a9..e4f5111e 100644 --- a/location/tests/test_finished_writes.py +++ b/location/tests/test_finished_writes.py @@ -41,6 +41,14 @@ FINISHED = "finished processing" + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + # FlowElements that only other FlowElements build on. BASES = set() @@ -88,6 +96,9 @@ def test_every_flow_element_refuses_writes_once_finished(self): with self.assertRaises(Exception) as caught: data.written_after_processing = True self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) processed.add(type(element)) self.assertEqual( defined_flow_elements(fiftyone_location) - BASES, processed) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_finished_writes.py index 520caa6b..dc85cf5e 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_finished_writes.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_finished_writes.py @@ -45,6 +45,15 @@ from .classes.cloudrequestengine_testbase import CloudRequestEngineTestsBase FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + EXAMPLES = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "examples", "src", "fiftyone_pipeline_examples", "cloud") @@ -94,6 +103,9 @@ def assert_writes_fail(self, flowdata, key): with self.assertRaises(Exception) as caught: data.written_after_processing = True self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) def test_every_flow_element_refuses_writes_once_finished(self): cloud = CloudRequestEngine({ diff --git a/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py index c58fdd2c..6c9575ec 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py @@ -28,9 +28,11 @@ FlowElements, then write to each one's ElementData and expect the error. """ +import copy import importlib import importlib.util import inspect +import json import os import pkgutil import unittest @@ -69,6 +71,43 @@ def process_internal(self, flowdata): data.written = self.value +class AddingElement(FlowElement): + """Adds data under another FlowElement's key, and a value to its + contents while it processes.""" + + def __init__(self): + super(AddingElement, self).__init__() + self.datakey = "adder" + + def process_internal(self, flowdata): + data = ElementDataDictionary(WritingElement("other"), {}) + data.contents["item"] = "while processing" + flowdata.set_element_data(data) + + +def item_store(data): + """The dictionary the data keeps its values in, or None.""" + if isinstance(data, dict): + return data + if (isinstance(getattr(type(data), "contents", None), property) + or "contents" in vars(data)): + return data.contents + return None + + +# Each way a dictionary can be written. +DICT_WRITES = ( + ("[]=", lambda d: d.__setitem__("b", 2)), + ("del", lambda d: d.__delitem__("a")), + ("update", lambda d: d.update(b=2)), + ("pop", lambda d: d.pop("a")), + ("popitem", lambda d: d.popitem()), + ("setdefault", lambda d: d.setdefault("b", 2)), + ("clear", lambda d: d.clear()), + ("|=", lambda d: d.__ior__({"b": 2})), +) + + def process(*elements, **settings): builder = PipelineBuilder(dict({"add_javascript_builder": False, "use_setheader_properties": False}, @@ -168,6 +207,34 @@ def test_the_data_keeps_its_own_state_once_finished(self): data._kept = "a value kept once read" self.assertEqual("a value kept once read", data._kept) + def test_the_contents_cannot_be_written_once_finished(self): + for name, write in DICT_WRITES: + with self.subTest(name): + data = process(WritingElement()).get("writer") + dict.__setitem__(data.contents, "a", 1) + for contents in (data.contents, data.as_dictionary()): + with self.assertRaises(Exception) as caught: + write(contents) + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual({"a": 1}, dict(data.contents)) + + def test_the_contents_cannot_be_replaced_once_finished(self): + data = process(WritingElement()).get("writer") + with self.assertRaises(Exception) as caught: + data.contents = {"b": 2} + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual({}, dict(data.contents)) + + def test_the_contents_are_a_dict_that_is_written_while_processing(self): + flowdata = process(AddingElement()) + contents = flowdata.get("other").contents + self.assertIsInstance(contents, dict) + self.assertEqual({"item": "while processing"}, contents) + self.assertEqual('{"item": "while processing"}', + json.dumps(contents)) + self.assertIs(dict, type(copy.deepcopy(contents))) + self.assertIs(dict, type(contents.copy())) + def test_closing_the_flow_data_is_not_a_write(self): class ClosingData(ElementDataDictionary): def close(self): @@ -193,6 +260,11 @@ def assert_writes_fail(self, flowdata, key): with self.assertRaises(Exception) as caught: data.written_after_processing = True self.assertIn(FINISHED, str(caught.exception)) + contents = item_store(data) + if contents is not None: + with self.assertRaises(Exception) as caught: + contents["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) def test_the_package_flow_elements_refuse_writes_once_finished(self): flowdata = PipelineBuilder().build().create_flowdata() diff --git a/pipeline/fiftyone_pipeline_derivedproperty/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_finished_writes.py index e2754fcd..d02452c1 100644 --- a/pipeline/fiftyone_pipeline_derivedproperty/tests/test_finished_writes.py +++ b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_finished_writes.py @@ -45,6 +45,15 @@ from .helpers import SourceElement, band_script, pipeline_of, processed FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + EXAMPLES = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "examples") @@ -77,6 +86,8 @@ def test_every_derived_element_writes_only_while_it_processes(): data.set_value("Demo", "written after processing") with pytest.raises(Exception, match=FINISHED): data.written_after_processing = True + with pytest.raises(Exception, match=FINISHED): + item_store(data)["written_after_processing"] = True assert data.get("Demo").value() == "High" assert defined_flow_elements(fiftyone_pipeline_derivedproperty) == \ {DerivedPropertyElement} @@ -89,4 +100,6 @@ def test_the_example_flow_element_refuses_writes_once_finished(): spec.loader.exec_module(module) flowdata = processed(pipeline_of([module.ScreenElement()])) with pytest.raises(Exception, match=FINISHED): - flowdata.get("screen").written_after_processing = True \ No newline at end of file + flowdata.get("screen").written_after_processing = True + with pytest.raises(Exception, match=FINISHED): + item_store(flowdata.get("screen"))["written_after_processing"] = True diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_engines/tests/test_finished_writes.py index 50640c72..fdb45ade 100644 --- a/pipeline/fiftyone_pipeline_engines/tests/test_finished_writes.py +++ b/pipeline/fiftyone_pipeline_engines/tests/test_finished_writes.py @@ -39,6 +39,15 @@ from fiftyone_pipeline_engines.engine import Engine FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + EXAMPLES = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "examples") @@ -79,6 +88,9 @@ def test_the_example_engine_refuses_writes_once_finished(self): with self.assertRaises(Exception) as caught: flowdata.get("astrology").written_after_processing = True self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(flowdata.get("astrology"))["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) if __name__ == "__main__": diff --git a/pipeline/fiftyone_pipeline_translation/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_translation/tests/test_finished_writes.py index b0d5674b..ac741857 100644 --- a/pipeline/fiftyone_pipeline_translation/tests/test_finished_writes.py +++ b/pipeline/fiftyone_pipeline_translation/tests/test_finished_writes.py @@ -42,6 +42,13 @@ FINISHED = "finished processing" +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + def defined_flow_elements(package): """Every FlowElement class defined in the package's own modules.""" found = set() @@ -80,6 +87,9 @@ def test_every_flow_element_refuses_writes_once_finished(self): with self.assertRaises(Exception) as caught: data.written_after_processing = True self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) self.assertEqual( {TranslationEngineBase}, defined_flow_elements(fiftyone_pipeline_translation)) From 5e070d80f6cf20eb131e6dc870b977151157e802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eddie=20Vereb=C3=A9lyi?= Date: Wed, 18 Nov 2026 22:17:27 +0000 Subject: [PATCH 04/11] Refuse dictionary writes to finished Element Data ElementDataDictionary and AspectDataDictionary, and so CloudData, keep their contents in a dict that checks the data is still being written before each write method, as setting an attribute does. Reads and the dict type are unchanged, and a copy is a plain dict. Setting contents keeps a checked copy. HardwareProfileData, a dict itself, checks its own write methods the same way. A profile is finished when the hardware data that holds it is, as the Flow Data records that only on the data it holds. --- .../hardwareprofile_cloud.py | 21 ++++- .../src/fiftyone_pipeline_core/elementdata.py | 80 +++++++++++++++++++ .../elementdata_dictionary.py | 5 +- .../aspectdata_dictionary.py | 5 ++ 4 files changed, 109 insertions(+), 2 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py index a19b7da2..cf388bd7 100644 --- a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py @@ -28,6 +28,7 @@ from fiftyone_pipeline_engines.aspectdata import AspectData from fiftyone_pipeline_engines.aspectdata_dictionary import AspectDataDictionary from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue +from fiftyone_pipeline_core.elementdata import _CheckedDictWrites import json @@ -60,7 +61,7 @@ def _cloud_response(flowdata): return data.get("cloud") -class HardwareProfileData(dict, AspectData): +class HardwareProfileData(_CheckedDictWrites, dict, AspectData): """! One hardware profile from a profile lookup, read in the way the device data is read. It is Aspect Data, so a property is read as an attribute, @@ -72,6 +73,10 @@ class HardwareProfileData(dict, AspectData): Attribute access, get(), [] and in compare names ignoring letter case. A property whose name is also a dict method, such as keys, is read with get() or []. + + A profile is part of the hardware data, and is written only while the + engine processes. Once it has finished, setting an attribute or writing + through a dict method raises an exception. """ # Attributes set in the constructor, which are never read as properties. @@ -93,6 +98,18 @@ def __init__(self, flow_element, contents): # reported in the same way for both. self.missing_property_service = CloudMissingPropertyService() + @property + def _finished(self): + # Finished when the hardware data that holds the profile is, as + # FlowData records that on the data it holds, not on each profile. + holder = self.__dict__.get("_holder") + return (self.__dict__.get("_own_finished", False) + or (holder is not None and holder._finished)) + + @_finished.setter + def _finished(self, finished): + self.__dict__["_own_finished"] = finished + def _matching_key(self, key): """! The key held for a property name given in another letter case, or @@ -247,6 +264,8 @@ def process_internal(self, flowdata): contents["profiles"] = devices data = AspectDataDictionary(self, contents) + for device in devices: + device._holder = data flowdata.set_element_data(data) \ No newline at end of file diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py index 19d42eaf..89856ebe 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py @@ -23,6 +23,86 @@ from .messages import Messages +class _CheckedDictWrites(object): + + """! + The write methods of a dict, each of which calls _check_writable first, + so a dict that holds Element Data values refuses writes once the + FlowElement that writes them has finished. Reads are the dict's own. + """ + + __slots__ = () + + def __setitem__(self, key, value): + self._check_writable(key) + super(_CheckedDictWrites, self).__setitem__(key, value) + + def __delitem__(self, key): + self._check_writable(key) + super(_CheckedDictWrites, self).__delitem__(key) + + def __ior__(self, other): + self._check_writable("|=") + return super(_CheckedDictWrites, self).__ior__(other) + + def update(self, *args, **kwargs): + self._check_writable("update") + super(_CheckedDictWrites, self).update(*args, **kwargs) + + def pop(self, *args): + self._check_writable(args[0] if args else "pop") + return super(_CheckedDictWrites, self).pop(*args) + + def popitem(self): + self._check_writable("popitem") + return super(_CheckedDictWrites, self).popitem() + + def setdefault(self, key, default=None): + self._check_writable(key) + return super(_CheckedDictWrites, self).setdefault(key, default) + + def clear(self): + self._check_writable("clear") + super(_CheckedDictWrites, self).clear() + + +class _ContentsDict(_CheckedDictWrites, dict): + + """! + The contents of Element Data that keeps its values in a dict. A write + raises once the FlowElement that writes the data has finished, as a + write to the data itself does. A copy is a plain dict. + """ + + __slots__ = ("_owner",) + + def __init__(self, owner, values): + dict.__init__(self, values) + self._owner = owner + + def _check_writable(self, name): + self._owner._check_writable(name) + + def __reduce_ex__(self, protocol): + return (dict, (dict(self),)) + + +def _get_contents(data): + return data.__dict__.get("_contents") + + +def _set_contents(data, values): + data._contents = _ContentsDict(data, values) + + +# The contents property of the Element Data classes that keep their values +# in a dict. Setting it keeps a copy, which refuses writes once finished. +_contents_property = property( + _get_contents, _set_contents, + doc="The values, keyed by property name. Written only while the " + "FlowElement that provides the data processes.") + + class ElementData(object): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_dictionary.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_dictionary.py index 1becbadb..961a722c 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_dictionary.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_dictionary.py @@ -20,7 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -from .elementdata import ElementData +from .elementdata import ElementData, _contents_property class ElementDataDictionary(ElementData): @@ -30,6 +30,9 @@ class ElementDataDictionary(ElementData): """ + # A dict that refuses writes once the FlowElement has finished. + contents = _contents_property + def __init__(self, flow_element, contents): """! diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata_dictionary.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata_dictionary.py index b259bdcb..88a6dd14 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata_dictionary.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata_dictionary.py @@ -20,6 +20,8 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from fiftyone_pipeline_core.elementdata import _contents_property + from .aspectdata import AspectData class AspectDataDictionary(AspectData): @@ -30,6 +32,9 @@ class AspectDataDictionary(AspectData): """ + # A dict that refuses writes once the FlowElement has finished. + contents = _contents_property + def __init__(self, flow_element, contents): """! From 0211ec6d3460b8f0966c2147e8336a65a5838f27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eddie=20Vereb=C3=A9lyi?= Date: Wed, 18 Nov 2026 22:18:01 +0000 Subject: [PATCH 05/11] Test that all Element Data in a processed Flow Data is finished Data added before process(), or added by a Flow Element under another Flow Element's key, was never recorded as finished, so it could still be written once the Flow Data had finished processing. --- .../tests/test_finished_writes.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py index 6c9575ec..ea59ee77 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py @@ -235,6 +235,30 @@ def test_the_contents_are_a_dict_that_is_written_while_processing(self): self.assertIs(dict, type(copy.deepcopy(contents))) self.assertIs(dict, type(contents.copy())) + def test_data_under_another_key_is_finished_too(self): + flowdata = process(AddingElement()) + data = flowdata.get("other") + with self.assertRaises(Exception) as caught: + data.written = "after" + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + data.contents["item"] = "after" + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual("while processing", data.contents["item"]) + + def test_data_added_before_processing_is_finished_with_it(self): + flowdata = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(WritingElement()).build().create_flowdata() + data = ElementDataDictionary(WritingElement("early"), {}) + flowdata.set_element_data(data) + data.written = "before processing" + flowdata.process() + with self.assertRaises(Exception) as caught: + data.written = "after" + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual("before processing", data.written) + def test_closing_the_flow_data_is_not_a_write(self): class ClosingData(ElementDataDictionary): def close(self): From a76863f77f9d1042c6a536299ee2e9ffabbe15b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eddie=20Vereb=C3=A9lyi?= Date: Wed, 18 Nov 2026 22:18:42 +0000 Subject: [PATCH 06/11] Finish all Element Data when a Flow Data finishes processing Once every Flow Element has run, process() records each Element Data the Flow Data holds as finished, including data added before processing and data a Flow Element added under another Flow Element's key. --- .../src/fiftyone_pipeline_core/flowdata.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py index caab8507..8821be03 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py @@ -104,6 +104,11 @@ def process(self): finally: _set_finished(self.data.get(key), True) + # Data added before processing, or under another FlowElement's key, + # is finished too. + for element_data in list(self.data.values()): + _set_finished(element_data, True) + # Set processed flag to true. flowdata can only be processed once, # and is read only from now on. self.processed = True From 839f7bce42decc1aa646dffc53a6c2230d4e60f3 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 19 Nov 2026 07:30:10 +0000 Subject: [PATCH 07/11] Test that a finished Flow Data refuses writes through its dictionaries Adding Element Data, errors or Evidence to a Flow Data that has finished processing must fail. These tests write through each public dictionary, being data, errors and the Evidence's values, and replace each of them, after processing, and expect the error. The two released set header tests now give their data to a Flow Data that has not been processed, because a finished one takes no more Element Data. --- .../tests/test_finished_writes.py | 80 +++++++++++++++++++ .../tests/test_setheader.py | 8 +- 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py index ea59ee77..90aa322b 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py @@ -40,6 +40,7 @@ import fiftyone_pipeline_core from fiftyone_pipeline_core.elementdata_dictionary import \ ElementDataDictionary +from fiftyone_pipeline_core.evidence import Evidence from fiftyone_pipeline_core.flowelement import FlowElement from fiftyone_pipeline_core.flowerror import FlowError from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder @@ -71,6 +72,17 @@ def process_internal(self, flowdata): data.written = self.value +class RaisingElement(FlowElement): + """Raises while it processes, so the FlowData records an error.""" + + def __init__(self, datakey="raiser"): + super(RaisingElement, self).__init__() + self.datakey = datakey + + def process_internal(self, flowdata): + raise Exception("raised while processing") + + class AddingElement(FlowElement): """Adds data under another FlowElement's key, and a value to its contents while it processes.""" @@ -182,6 +194,74 @@ def test_evidence_can_be_added_before_processing(self): flowdata.evidence.add("query.early", "value") self.assertEqual("value", flowdata.evidence.get("query.early")) + def assert_dict_writes_fail(self, finished, store): + """Each way of writing the dictionary store(finished) raises, and + leaves it as it was.""" + for name, write in DICT_WRITES: + with self.subTest(name): + flowdata = finished() + before = dict(store(flowdata)) + with self.assertRaises(Exception) as caught: + write(store(flowdata)) + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual(before, dict(store(flowdata))) + + def test_the_element_data_dictionary_cannot_be_written_once_processed( + self): + self.assert_dict_writes_fail( + lambda: process(WritingElement("a")), lambda f: f.data) + + def test_the_errors_dictionary_cannot_be_written_once_processed(self): + self.assert_dict_writes_fail( + lambda: process(RaisingElement("a"), + suppress_process_exceptions=True), + lambda f: f.errors) + + def test_the_evidence_dictionaries_cannot_be_written_once_processed( + self): + def finished(): + flowdata = PipelineBuilder().add(WritingElement()).build() \ + .create_flowdata() + flowdata.evidence.add("a", "value") + flowdata.process() + return flowdata + + for store in (lambda f: f.evidence.get_all(), + lambda f: f.evidence.evidence, + lambda f: f.evidence.get_unaccepted()): + self.assert_dict_writes_fail(finished, store) + + def test_the_dictionaries_cannot_be_replaced_once_processed(self): + flowdata = process(WritingElement()) + evidence = flowdata.evidence + for owner, name, value in ( + (flowdata, "data", {}), + (flowdata, "errors", {}), + (flowdata, "evidence", Evidence(flowdata)), + (evidence, "evidence", {}), + (evidence, "unaccepted", {})): + with self.subTest(name): + before = getattr(owner, name) + with self.assertRaises(Exception) as caught: + setattr(owner, name, value) + self.assertIn(FINISHED, str(caught.exception)) + self.assertIs(before, getattr(owner, name)) + + def test_data_given_before_processing_is_read_only_after(self): + flowdata = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(WritingElement()).build().create_flowdata() + flowdata.data = {"given": "before processing"} + flowdata.errors = {} + flowdata.data["added"] = "before processing" + flowdata.process() + self.assertEqual("before processing", flowdata.get("given")) + self.assertEqual("before processing", flowdata.get("added")) + for store in (flowdata.data, flowdata.errors): + with self.assertRaises(Exception) as caught: + store["later"] = "after processing" + self.assertIn(FINISHED, str(caught.exception)) + class FinishedElementDataTests(unittest.TestCase): diff --git a/pipeline/fiftyone_pipeline_core/tests/test_setheader.py b/pipeline/fiftyone_pipeline_core/tests/test_setheader.py index 9f96df6a..67311854 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_setheader.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_setheader.py @@ -57,7 +57,9 @@ class SetHeaderUACHTests(unittest.TestCase): def test_get_response_header_value(self, device, expected_value): testPipeline = TestPipeline() setHeaderElement = SetHeaderElement() - flowdata = testPipeline.flowdata + # A FlowData that has not been processed, because a finished one + # takes no more ElementData. + flowdata = testPipeline.pipeline.create_flowdata() flowdata.data = device actual_value = setHeaderElement.get_response_header_value(flowdata, setHeader_properties_dict) self.assertEqual(expected_value, actual_value[Constants.ACCEPTCH_HEADER]) @@ -80,7 +82,9 @@ def test_set_response_header(self, data, expected_value): response = session.get('mock://test.com/3') testPipeline = TestPipeline() - flowdata = testPipeline.flowdata + # A FlowData that has not been processed, because a finished one + # takes no more ElementData. + flowdata = testPipeline.pipeline.create_flowdata() flowdata.data = data response = set_response_header(flowdata, response) From 6f1b29faedbb54659af845a1c73b6fabef65596f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 19 Nov 2026 07:32:02 +0000 Subject: [PATCH 08/11] Refuse writes through a finished Flow Data's dictionaries A Flow Data now keeps its Element Data, its errors and its Evidence values in dicts whose write methods raise once processing has finished, and data, errors and evidence cannot be replaced then. The check reads the processed field and takes no lock. add_from_dict also takes any dict, such as the Evidence of another Flow Data. --- .../src/fiftyone_pipeline_core/elementdata.py | 25 ++++++++++++++ .../src/fiftyone_pipeline_core/evidence.py | 27 +++++++++++++-- .../src/fiftyone_pipeline_core/flowdata.py | 33 ++++++++++++++++++- 3 files changed, 82 insertions(+), 3 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py index 89856ebe..f7fca9de 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py @@ -87,6 +87,31 @@ def __reduce_ex__(self, protocol): return (dict, (dict(self),)) +class _FlowDataDict(_CheckedDictWrites, dict): + + """! + A dict that holds what a FlowData has been given, being its ElementData, + its errors or its Evidence. A write raises once the FlowData has + finished processing. A copy is a plain dict. + """ + + __slots__ = ("_flowdata", "_what") + + def __init__(self, flowdata, what, values=()): + dict.__init__(self, values) + self._flowdata = flowdata + self._what = what + + def _check_writable(self, name): + # Read from the FlowData's own fields, because FlowData treats an + # attribute it does not have as the key of ElementData. + if getattr(self._flowdata, "__dict__", {}).get("processed"): + raise Exception(Messages.FLOW_DATA_FINISHED.format(self._what)) + + def __reduce_ex__(self, protocol): + return (dict, (dict(self),)) + + def _get_contents(data): return data.__dict__.get("_contents") diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py index 4f59b61d..290b1af1 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py @@ -20,6 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from .elementdata import _FlowDataDict from .messages import Messages # The key the web integration gives the web request itself, which the usage @@ -27,6 +28,9 @@ # sets it aside, whatever the elements' filters accept. HTTP_REQUEST_KEY = "server.http-request" +# The attributes that hold Evidence values. +_VALUES = ("evidence", "unaccepted") + class Evidence: def __init__(self, flowdata): @@ -36,14 +40,33 @@ def __init__(self, flowdata): @type flowdata: FlowData @param flowdata: Parent FlowData """ + self.flowdata = flowdata self.evidence = {} # Each key in lower case to the key as first added, because keys # ignore letter case. self.keys = {} self.evidence_filtered = False - self.flowdata = flowdata self.unaccepted = {} + def __setattr__(self, name, value): + + """! + Set an attribute. The Evidence values, and the values filter() sets + aside, are kept in dicts that refuse writes once the FlowData has + finished processing, and neither can be replaced then. + + @raise Exception: when one of them is replaced once the FlowData + has finished processing + """ + + if name in _VALUES: + flowdata = self.__dict__.get("flowdata") + if name in self.__dict__ and \ + getattr(flowdata, "__dict__", {}).get("processed"): + raise Exception(Messages.FLOW_DATA_FINISHED.format("Evidence")) + value = _FlowDataDict(flowdata, "Evidence", value) + object.__setattr__(self, name, value) + def add(self, key, value): """! Add a single piece of evidence by its element and value @@ -78,7 +101,7 @@ def add_from_dict(self, evidence_dictionary): if self.flowdata.processed: raise Exception(Messages.FLOW_DATA_FINISHED.format("Evidence")) - if not type(evidence_dictionary) is dict: + if not isinstance(evidence_dictionary, dict): self.flowdata.set_error("core", "Must pass valid dictionary.") for key, value in evidence_dictionary.items(): diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py index babeba1b..be0f3d82 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py @@ -20,7 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -from .elementdata import ElementData +from .elementdata import ElementData, _FlowDataDict from .evidence import Evidence from .flowerror import FlowError from .messages import Messages @@ -36,6 +36,11 @@ def _set_finished(element_data, finished): object.__setattr__(element_data, "_finished", finished) +# The attributes that hold what a FlowData has been given, each with what +# it holds for the message when a finished FlowData is written. +_GIVEN = {"data": "ElementData", "errors": "an error", "evidence": "Evidence"} + + class FlowData: """! FlowData is created by a specific Pipeline @@ -50,6 +55,11 @@ class FlowData: request, uses it only in the code that handles that request, and closes it when the request ends. + Once processing has finished, a FlowData is read only. Adding Evidence, + ElementData or errors to it raises, whether through its methods or + through the dicts data, errors and evidence.get_all() return. Only + process() sets processed, and setting it back is not supported. + """ def __init__(self, pipeline): @@ -69,6 +79,27 @@ def __init__(self, pipeline): self.stopped = False self.evidence = Evidence(self) + def __setattr__(self, name, value): + + """! + Set an attribute. The ElementData and errors are kept in dicts that + refuse writes once processing has finished, and they and the + Evidence cannot be replaced then. + + @raise Exception: when one of them is set once processing has + finished + """ + + if name in _GIVEN: + # Read from the instance's own fields, because a missing + # attribute is treated as the key of ElementData. + if self.__dict__.get("processed"): + raise Exception(Messages.FLOW_DATA_FINISHED.format( + _GIVEN[name])) + if name != "evidence": + value = _FlowDataDict(self, _GIVEN[name], value) + object.__setattr__(self, name, value) + def process(self): """! From 7835b7bddaa4420766bb54454984a5a131b5884c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 19 Nov 2026 07:32:56 +0000 Subject: [PATCH 09/11] Test that finished Element Data can be closed directly Closing Element Data releases what it holds, which is not a write. These tests close finished data directly, an on-premise engine's results in each on-premise package and a class whose close sets a public attribute in the core, and expect the close to work and the data to stay finished. --- .../tests/test_finished_writes.py | 20 ++++++++++ .../tests/test_finished_writes.py | 20 ++++++++++ .../tests/test_finished_writes.py | 40 +++++++++++++------ 3 files changed, 68 insertions(+), 12 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_finished_writes.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_finished_writes.py index bcad8a34..3bb1cb5a 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_finished_writes.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_finished_writes.py @@ -32,6 +32,7 @@ import unittest import fiftyone_devicedetection_onpremise +from fiftyone_devicedetection_onpremise.constants import RESULTS_CLOSED from fiftyone_devicedetection_onpremise.devicedetection_onpremise import \ DeviceDetectionOnPremise from fiftyone_pipeline_core.flowelement import FlowElement @@ -76,6 +77,25 @@ def test_every_flow_element_refuses_writes_once_finished(self): {DeviceDetectionOnPremise}, defined_flow_elements(fiftyone_devicedetection_onpremise)) + def test_the_data_can_be_closed_once_finished(self): + # Closing releases the native results, which is not a write, so it + # works on finished data, as closing the FlowData does. + engine = DeviceDetectionOnPremise( + data_file_path=DATA_FILE, licence_keys="", auto_update=False) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("header.user-agent", USER_AGENT) + flowdata.process() + data = flowdata.get("device") + data.close() + with self.assertRaises(Exception) as caught: + data.get("ismobile") + self.assertEqual(RESULTS_CLOSED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + data.ismobile = False + self.assertIn(FINISHED, str(caught.exception)) + if __name__ == "__main__": unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_finished_writes.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_finished_writes.py index 251e8b9e..4474ad83 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_finished_writes.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_finished_writes.py @@ -32,6 +32,7 @@ import unittest import fiftyone_ipintelligence_onpremise +from fiftyone_ipintelligence_onpremise.constants import RESULTS_CLOSED from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise import \ IpIntelligenceOnPremise from fiftyone_pipeline_core.flowelement import FlowElement @@ -76,6 +77,25 @@ def test_every_flow_element_refuses_writes_once_finished(self): {IpIntelligenceOnPremise}, defined_flow_elements(fiftyone_ipintelligence_onpremise)) + def test_the_data_can_be_closed_once_finished(self): + # Closing releases the native results, which is not a write, so it + # works on finished data, as closing the FlowData does. + engine = IpIntelligenceOnPremise( + data_file_path=DATA_FILE, licence_keys="", auto_update=False) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("query.client-ip", "185.28.167.77") + flowdata.process() + data = flowdata.get("ip") + data.close() + with self.assertRaises(Exception) as caught: + data.get("registeredname") + self.assertEqual(RESULTS_CLOSED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + data.registeredname = "written after processing" + self.assertIn(FINISHED, str(caught.exception)) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py index 90aa322b..39d7ba75 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py @@ -97,6 +97,25 @@ def process_internal(self, flowdata): flowdata.set_element_data(data) +class ClosingData(ElementDataDictionary): + """Sets a public attribute when it is closed, as a close method that + releases what the data holds may.""" + + def close(self): + self.closed = True + + +class ClosingElement(FlowElement): + """Adds data that has a close method of its own.""" + + def __init__(self): + super(ClosingElement, self).__init__() + self.datakey = "closing" + + def process_internal(self, flowdata): + flowdata.set_element_data(ClosingData(self, {})) + + def item_store(data): """The dictionary the data keeps its values in, or None.""" if isinstance(data, dict): @@ -340,22 +359,19 @@ def test_data_added_before_processing_is_finished_with_it(self): self.assertEqual("before processing", data.written) def test_closing_the_flow_data_is_not_a_write(self): - class ClosingData(ElementDataDictionary): - def close(self): - self.closed = True - - class ClosingElement(FlowElement): - def __init__(self): - super(ClosingElement, self).__init__() - self.datakey = "closing" - - def process_internal(self, flowdata): - flowdata.set_element_data(ClosingData(self, {})) - flowdata = process(ClosingElement()) flowdata.close() self.assertTrue(flowdata.get("closing").closed) + def test_closing_the_data_itself_is_not_a_write(self): + data = process(ClosingElement()).get("closing") + data.close() + self.assertTrue(data.closed) + with self.assertRaises(Exception) as caught: + data.closed = False + self.assertIn(FINISHED, str(caught.exception)) + self.assertTrue(data.closed) + class EveryFlowElementTests(unittest.TestCase): From 90471f0e4cedd30b2c6db35b8d313581c37d968d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 19 Nov 2026 07:34:18 +0000 Subject: [PATCH 10/11] Let finished Element Data be closed directly Closing Element Data releases what it holds and is not a write. Each close method an Element Data class defines or inherits now runs with the data open for writing and finished again afterwards, so calling close() on finished on-premise results releases them instead of raising. FlowData.close relies on the same and no longer opens the data itself. --- .../src/fiftyone_pipeline_core/elementdata.py | 35 +++++++++++++++++++ .../src/fiftyone_pipeline_core/flowdata.py | 11 ++---- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py index f7fca9de..4d479bff 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py @@ -20,6 +20,9 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +import functools +import inspect + from .messages import Messages @@ -128,6 +131,23 @@ def _set_contents(data, values): "FlowElement that provides the data processes.") +def _open_while_closing(close): + + # Releasing what the data holds is not a write to it, so a close method + # runs with the data open for writing, and finished again afterwards. + @functools.wraps(close) + def opened(self, *args, **kwargs): + finished = self._finished + object.__setattr__(self, "_finished", False) + try: + return close(self, *args, **kwargs) + finally: + object.__setattr__(self, "_finished", finished) + + opened._opens_data = True + return opened + + class ElementData(object): """! @@ -142,8 +162,23 @@ class ElementData(object): instance that is written from several threads, or read while another thread writes it, is not catered for. + A subclass that holds something to release, such as native results, + gives itself a close method. Closing is not a write, so close may set + the data's attributes even once it is finished, whether it is called + directly or by FlowData.close. + """ + def __init_subclass__(cls, **kwargs): + + # A close method a subclass defines or inherits runs with the data + # open. Read without binding, so a static method is left alone. + super(ElementData, cls).__init_subclass__(**kwargs) + close = inspect.getattr_static(cls, "close", None) + if inspect.isfunction(close) and \ + not getattr(close, "_opens_data", False): + cls.close = _open_while_closing(close) + def __init__(self, flow_element): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py index be0f3d82..d3a23769 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py @@ -178,14 +178,9 @@ def close(self): # attribute it does not have as the name of a property. close = getattr(type(element_data), "close", None) if callable(close): - # Releasing what the data holds is not a write to it. - finished = (isinstance(element_data, ElementData) - and element_data._finished) - _set_finished(element_data, False) - try: - close(element_data) - finally: - _set_finished(element_data, finished) + # Closing is not a write, and ElementData opens itself for + # its close method. + close(element_data) def __enter__(self): return self From 2bc91de91fd3276eda9571f8d496c31e06c6372f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 19 Nov 2026 08:04:43 +0000 Subject: [PATCH 11/11] Tell pylint the dict write mixin's methods come from the class it is mixed into pylint reported no-member for each write method of the mixin, because _check_writable and the dict methods come from the class that mixes it in, and the package's lint fails on any error. --- .../src/fiftyone_pipeline_core/elementdata.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py index 4d479bff..085f03e8 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py @@ -34,6 +34,10 @@ class _CheckedDictWrites(object): FlowElement that writes them has finished. Reads are the dict's own. """ + # The class this is mixed into gives _check_writable and the dict's own + # methods, which pylint cannot see from here. + # pylint: disable=no-member + __slots__ = () def __setitem__(self, key, value):