diff --git a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py index a19b7da2..cf388bd7 100644 --- a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py @@ -28,6 +28,7 @@ from fiftyone_pipeline_engines.aspectdata import AspectData from fiftyone_pipeline_engines.aspectdata_dictionary import AspectDataDictionary from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue +from fiftyone_pipeline_core.elementdata import _CheckedDictWrites import json @@ -60,7 +61,7 @@ def _cloud_response(flowdata): return data.get("cloud") -class HardwareProfileData(dict, AspectData): +class HardwareProfileData(_CheckedDictWrites, dict, AspectData): """! One hardware profile from a profile lookup, read in the way the device data is read. It is Aspect Data, so a property is read as an attribute, @@ -72,6 +73,10 @@ class HardwareProfileData(dict, AspectData): Attribute access, get(), [] and in compare names ignoring letter case. A property whose name is also a dict method, such as keys, is read with get() or []. + + A profile is part of the hardware data, and is written only while the + engine processes. Once it has finished, setting an attribute or writing + through a dict method raises an exception. """ # Attributes set in the constructor, which are never read as properties. @@ -93,6 +98,18 @@ def __init__(self, flow_element, contents): # reported in the same way for both. self.missing_property_service = CloudMissingPropertyService() + @property + def _finished(self): + # Finished when the hardware data that holds the profile is, as + # FlowData records that on the data it holds, not on each profile. + holder = self.__dict__.get("_holder") + return (self.__dict__.get("_own_finished", False) + or (holder is not None and holder._finished)) + + @_finished.setter + def _finished(self, finished): + self.__dict__["_own_finished"] = finished + def _matching_key(self, key): """! The key held for a property name given in another letter case, or @@ -247,6 +264,8 @@ def process_internal(self, flowdata): contents["profiles"] = devices data = AspectDataDictionary(self, contents) + for device in devices: + device._holder = data flowdata.set_element_data(data) \ No newline at end of file diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_finished_writes.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_finished_writes.py new file mode 100644 index 00000000..4d0c9e49 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_finished_writes.py @@ -0,0 +1,140 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. +""" + +import importlib +import inspect +import json +import pkgutil +import unittest + +import fiftyone_devicedetection_cloud +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_devicedetection_cloud.devicedetection_cloud import \ + DeviceDetectionCloud +from fiftyone_devicedetection_cloud.hardwareprofile_cloud import \ + HardwareProfileCloud + +FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + +# FlowElements that only other FlowElements build on. +BASES = set() + + +class StandInRequestEngine(FlowElement): + """Stands in for a Cloud Request Engine whose answer has an empty + section for each of the given data keys.""" + + def __init__(self, keys): + super(StandInRequestEngine, self).__init__() + self.datakey = "cloud" + self.flow_element_properties = {key: {} for key in keys} + self.answer = json.dumps({key: {} for key in keys}) + + def process_internal(self, flowdata): + flowdata.set_element_data( + ElementDataDictionary(self, {"cloud": self.answer})) + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + processed = set() + for element in [DeviceDetectionCloud(), HardwareProfileCloud()]: + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + flowdata = builder \ + .add(StandInRequestEngine([element.datakey])) \ + .add(element).build().create_flowdata() + flowdata.process() + data = flowdata.get(element.datakey) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) + processed.add(type(element)) + self.assertEqual( + defined_flow_elements(fiftyone_devicedetection_cloud) - BASES, processed) + + def test_a_hardware_profile_refuses_writes_once_finished(self): + writes = ( + ("attribute", lambda p: setattr(p, "written", True)), + ("[]=", lambda p: p.__setitem__("written", True)), + ("del", lambda p: p.__delitem__("hardwarevendor")), + ("update", lambda p: p.update(written=True)), + ("pop", lambda p: p.pop("hardwarevendor")), + ("popitem", lambda p: p.popitem()), + ("setdefault", lambda p: p.setdefault("written", True)), + ("clear", lambda p: p.clear()), + ("|=", lambda p: p.__ior__({"written": True})), + ) + element = HardwareProfileCloud() + request = StandInRequestEngine([element.datakey]) + request.answer = json.dumps({element.datakey: { + "profiles": [{"hardwarevendor": "Apple"}]}}) + flowdata = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(request).add(element).build().create_flowdata() + flowdata.process() + profile = flowdata.get(element.datakey).get("profiles")[0] + for name, write in writes: + with self.subTest(name): + with self.assertRaises(Exception) as caught: + write(profile) + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual(["hardwarevendor"], list(profile)) + self.assertEqual("Apple", profile.hardwarevendor.value()) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_finished_writes.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_finished_writes.py new file mode 100644 index 00000000..3bb1cb5a --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_finished_writes.py @@ -0,0 +1,101 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. Reading values from the results is not a write. +""" + +import importlib +import inspect +import pkgutil +import unittest + +import fiftyone_devicedetection_onpremise +from fiftyone_devicedetection_onpremise.constants import RESULTS_CLOSED +from fiftyone_devicedetection_onpremise.devicedetection_onpremise import \ + DeviceDetectionOnPremise +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +DATA_FILE = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash" +USER_AGENT = ("Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) " + "AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148") +FINISHED = "finished processing" + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + engine = DeviceDetectionOnPremise( + data_file_path=DATA_FILE, licence_keys="", auto_update=False) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("header.user-agent", USER_AGENT) + flowdata.process() + data = flowdata.get("device") + self.assertTrue(data.ismobile.value()) + with self.assertRaises(Exception) as caught: + data.ismobile = False + self.assertIn(FINISHED, str(caught.exception)) + self.assertTrue(data.ismobile.value()) + self.assertEqual( + {DeviceDetectionOnPremise}, + defined_flow_elements(fiftyone_devicedetection_onpremise)) + + def test_the_data_can_be_closed_once_finished(self): + # Closing releases the native results, which is not a write, so it + # works on finished data, as closing the FlowData does. + engine = DeviceDetectionOnPremise( + data_file_path=DATA_FILE, licence_keys="", auto_update=False) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("header.user-agent", USER_AGENT) + flowdata.process() + data = flowdata.get("device") + data.close() + with self.assertRaises(Exception) as caught: + data.get("ismobile") + self.assertEqual(RESULTS_CLOSED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + data.ismobile = False + self.assertIn(FINISHED, str(caught.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_cloud/tests/test_finished_writes.py b/ip-intelligence/fiftyone_ipintelligence_cloud/tests/test_finished_writes.py new file mode 100644 index 00000000..4ab707da --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_cloud/tests/test_finished_writes.py @@ -0,0 +1,113 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. +""" + +import importlib +import inspect +import json +import pkgutil +import unittest + +import fiftyone_ipintelligence_cloud +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_ipintelligence_cloud.countries_translation_cloud import \ + CountriesTranslationCloud +from fiftyone_ipintelligence_cloud.ipintelligence_cloud import \ + IpIntelligenceCloud +from fiftyone_ipintelligence_cloud.weighted_cloud_engine import \ + WeightedCloudEngine + +FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + +# FlowElements that only other FlowElements build on. +BASES = {WeightedCloudEngine} + + +class StandInRequestEngine(FlowElement): + """Stands in for a Cloud Request Engine whose answer has an empty + section for each of the given data keys.""" + + def __init__(self, keys): + super(StandInRequestEngine, self).__init__() + self.datakey = "cloud" + self.flow_element_properties = {key: {} for key in keys} + self.answer = json.dumps({key: {} for key in keys}) + + def process_internal(self, flowdata): + flowdata.set_element_data( + ElementDataDictionary(self, {"cloud": self.answer})) + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + processed = set() + for element in [IpIntelligenceCloud(), CountriesTranslationCloud()]: + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + flowdata = builder \ + .add(StandInRequestEngine([element.datakey])) \ + .add(element).build().create_flowdata() + flowdata.process() + data = flowdata.get(element.datakey) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) + processed.add(type(element)) + self.assertEqual( + defined_flow_elements(fiftyone_ipintelligence_cloud) - BASES, processed) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_finished_writes.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_finished_writes.py new file mode 100644 index 00000000..4474ad83 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_finished_writes.py @@ -0,0 +1,101 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. Reading values from the results is not a write. +""" + +import importlib +import inspect +import pkgutil +import unittest + +import fiftyone_ipintelligence_onpremise +from fiftyone_ipintelligence_onpremise.constants import RESULTS_CLOSED +from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise import \ + IpIntelligenceOnPremise +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +from .constants import TestConstants + +DATA_FILE = TestConstants.get_data_file_path() +FINISHED = "finished processing" + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + engine = IpIntelligenceOnPremise( + data_file_path=DATA_FILE, licence_keys="", auto_update=False) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("query.client-ip", "185.28.167.77") + flowdata.process() + data = flowdata.get("ip") + name = data.get("registeredname") + with self.assertRaises(Exception) as caught: + data.registeredname = "written after processing" + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual(name.value(), data.get("registeredname").value()) + self.assertEqual( + {IpIntelligenceOnPremise}, + defined_flow_elements(fiftyone_ipintelligence_onpremise)) + + def test_the_data_can_be_closed_once_finished(self): + # Closing releases the native results, which is not a write, so it + # works on finished data, as closing the FlowData does. + engine = IpIntelligenceOnPremise( + data_file_path=DATA_FILE, licence_keys="", auto_update=False) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("query.client-ip", "185.28.167.77") + flowdata.process() + data = flowdata.get("ip") + data.close() + with self.assertRaises(Exception) as caught: + data.get("registeredname") + self.assertEqual(RESULTS_CLOSED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + data.registeredname = "written after processing" + self.assertIn(FINISHED, str(caught.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_finished_writes.py b/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_finished_writes.py new file mode 100644 index 00000000..26e75e85 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_finished_writes.py @@ -0,0 +1,99 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. +""" + +import importlib +import inspect +import pkgutil +import unittest + +import fiftyone_ipintelligence_translation +from fiftyone_ipintelligence_translation.countries_translation_engine import \ + CountriesTranslationEngine +from fiftyone_ipintelligence_translation.countries_translation_engine_builder \ + import CountriesTranslationEngineBuilder +from fiftyone_ipintelligence_translation.country_code_translation_engine \ + import CountryCodeTranslationEngine +from fiftyone_ipintelligence_translation \ + .country_code_translation_engine_builder import \ + CountryCodeTranslationEngineBuilder +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +from .mock_ip_engine import mock_ip_element + +FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + engines = [CountryCodeTranslationEngineBuilder().build(), + CountriesTranslationEngineBuilder().build()] + builder = PipelineBuilder().add(mock_ip_element(["DE"], ["DE"])) + for engine in engines: + builder.add(engine) + flowdata = builder.build().create_flowdata() + flowdata.process() + for engine in engines: + data = flowdata.get(engine.datakey) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual( + {type(engine) for engine in engines}, + defined_flow_elements(fiftyone_ipintelligence_translation)) + self.assertEqual( + {CountriesTranslationEngine, CountryCodeTranslationEngine}, + {type(engine) for engine in engines}) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/location/tests/test_finished_writes.py b/location/tests/test_finished_writes.py new file mode 100644 index 00000000..e4f5111e --- /dev/null +++ b/location/tests/test_finished_writes.py @@ -0,0 +1,108 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. +""" + +import importlib +import inspect +import json +import pkgutil +import unittest + +import fiftyone_location +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_location.location_cloud import LocationCloud + +FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + +# FlowElements that only other FlowElements build on. +BASES = set() + + +class StandInRequestEngine(FlowElement): + """Stands in for a Cloud Request Engine whose answer has an empty + section for each of the given data keys.""" + + def __init__(self, keys): + super(StandInRequestEngine, self).__init__() + self.datakey = "cloud" + self.flow_element_properties = {key: {} for key in keys} + self.answer = json.dumps({key: {} for key in keys}) + + def process_internal(self, flowdata): + flowdata.set_element_data( + ElementDataDictionary(self, {"cloud": self.answer})) + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + processed = set() + for element in [LocationCloud()]: + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + flowdata = builder \ + .add(StandInRequestEngine([element.datakey])) \ + .add(element).build().create_flowdata() + flowdata.process() + data = flowdata.get(element.datakey) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) + processed.add(type(element)) + self.assertEqual( + defined_flow_elements(fiftyone_location) - BASES, processed) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_finished_writes.py new file mode 100644 index 00000000..dc85cf5e --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_finished_writes.py @@ -0,0 +1,147 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines, and every one its examples define, is +processed, then a write to its ElementData is expected to fail, as the thread +safety page of the specification requires. +""" + +import importlib +import importlib.util +import inspect +import json +import os +import pkgutil + +import fiftyone_pipeline_cloudrequestengine +from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine +from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import \ + CloudRequestEngine +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +from .classes.cloudrequestengine_testbase import CloudRequestEngineTestsBase + +FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + +EXAMPLES = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "examples", "src", "fiftyone_pipeline_examples", "cloud") + + +class StandInRequestEngine(FlowElement): + """Stands in for a Cloud Request Engine whose answer has an empty + section for each of the given data keys.""" + + def __init__(self, keys): + super(StandInRequestEngine, self).__init__() + self.datakey = "cloud" + self.flow_element_properties = {key: {} for key in keys} + self.answer = json.dumps({key: {} for key in keys}) + + def process_internal(self, flowdata): + flowdata.set_element_data( + ElementDataDictionary(self, {"cloud": self.answer})) + + +def load_example(name): + """An example module, loaded from its file under its own name.""" + spec = importlib.util.spec_from_file_location( + "example_" + name, os.path.join(EXAMPLES, name + ".py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(CloudRequestEngineTestsBase): + + def assert_writes_fail(self, flowdata, key): + data = flowdata.get(key) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) + + def test_every_flow_element_refuses_writes_once_finished(self): + cloud = CloudRequestEngine({ + "license_key": "a-licence-key", + "requested_properties": ["device.ismobile"], + "http_client": self.mock_http(), + }) + engine = CloudEngine() + engine.datakey = "device" + flowdata = PipelineBuilder().add(cloud).add(engine).build() \ + .create_flowdata() + flowdata.evidence.add("query.User-Agent", "iPhone") + flowdata.process() + self.assert_writes_fail(flowdata, "cloud") + self.assert_writes_fail(flowdata, "device") + self.assertEqual( + {CloudEngine, CloudRequestEngine}, + defined_flow_elements(fiftyone_pipeline_cloudrequestengine)) + + def test_every_example_flow_element_refuses_writes_once_finished(self): + engines = load_example("engines") + simple = load_example("simple_cloud_engine") + elements = [engines.DeviceDetectionCloud(), + engines.IpIntelligenceCloud(), + engines.CountriesTranslationCloud(), + simple.SimpleCloudEngine()] + for element in elements: + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + flowdata = builder \ + .add(StandInRequestEngine([element.datakey])) \ + .add(element).build().create_flowdata() + flowdata.process() + self.assert_writes_fail(flowdata, element.datakey) + + +if __name__ == "__main__": + import unittest + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py index 5ee546d3..085f03e8 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py @@ -20,6 +20,137 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +import functools +import inspect + +from .messages import Messages + + +class _CheckedDictWrites(object): + + """! + The write methods of a dict, each of which calls _check_writable first, + so a dict that holds Element Data values refuses writes once the + FlowElement that writes them has finished. Reads are the dict's own. + """ + + # The class this is mixed into gives _check_writable and the dict's own + # methods, which pylint cannot see from here. + # pylint: disable=no-member + + __slots__ = () + + def __setitem__(self, key, value): + self._check_writable(key) + super(_CheckedDictWrites, self).__setitem__(key, value) + + def __delitem__(self, key): + self._check_writable(key) + super(_CheckedDictWrites, self).__delitem__(key) + + def __ior__(self, other): + self._check_writable("|=") + return super(_CheckedDictWrites, self).__ior__(other) + + def update(self, *args, **kwargs): + self._check_writable("update") + super(_CheckedDictWrites, self).update(*args, **kwargs) + + def pop(self, *args): + self._check_writable(args[0] if args else "pop") + return super(_CheckedDictWrites, self).pop(*args) + + def popitem(self): + self._check_writable("popitem") + return super(_CheckedDictWrites, self).popitem() + + def setdefault(self, key, default=None): + self._check_writable(key) + return super(_CheckedDictWrites, self).setdefault(key, default) + + def clear(self): + self._check_writable("clear") + super(_CheckedDictWrites, self).clear() + + +class _ContentsDict(_CheckedDictWrites, dict): + + """! + The contents of Element Data that keeps its values in a dict. A write + raises once the FlowElement that writes the data has finished, as a + write to the data itself does. A copy is a plain dict. + """ + + __slots__ = ("_owner",) + + def __init__(self, owner, values): + dict.__init__(self, values) + self._owner = owner + + def _check_writable(self, name): + self._owner._check_writable(name) + + def __reduce_ex__(self, protocol): + return (dict, (dict(self),)) + + +class _FlowDataDict(_CheckedDictWrites, dict): + + """! + A dict that holds what a FlowData has been given, being its ElementData, + its errors or its Evidence. A write raises once the FlowData has + finished processing. A copy is a plain dict. + """ + + __slots__ = ("_flowdata", "_what") + + def __init__(self, flowdata, what, values=()): + dict.__init__(self, values) + self._flowdata = flowdata + self._what = what + + def _check_writable(self, name): + # Read from the FlowData's own fields, because FlowData treats an + # attribute it does not have as the key of ElementData. + if getattr(self._flowdata, "__dict__", {}).get("processed"): + raise Exception(Messages.FLOW_DATA_FINISHED.format(self._what)) + + def __reduce_ex__(self, protocol): + return (dict, (dict(self),)) + + +def _get_contents(data): + return data.__dict__.get("_contents") + + +def _set_contents(data, values): + data._contents = _ContentsDict(data, values) + + +# The contents property of the Element Data classes that keep their values +# in a dict. Setting it keeps a copy, which refuses writes once finished. +_contents_property = property( + _get_contents, _set_contents, + doc="The values, keyed by property name. Written only while the " + "FlowElement that provides the data processes.") + + +def _open_while_closing(close): + + # Releasing what the data holds is not a write to it, so a close method + # runs with the data open for writing, and finished again afterwards. + @functools.wraps(close) + def opened(self, *args, **kwargs): + finished = self._finished + object.__setattr__(self, "_finished", False) + try: + return close(self, *args, **kwargs) + finally: + object.__setattr__(self, "_finished", finished) + + opened._opens_data = True + return opened + class ElementData(object): @@ -35,8 +166,23 @@ class ElementData(object): instance that is written from several threads, or read while another thread writes it, is not catered for. + A subclass that holds something to release, such as native results, + gives itself a close method. Closing is not a write, so close may set + the data's attributes even once it is finished, whether it is called + directly or by FlowData.close. + """ + def __init_subclass__(cls, **kwargs): + + # A close method a subclass defines or inherits runs with the data + # open. Read without binding, so a static method is left alone. + super(ElementData, cls).__init_subclass__(**kwargs) + close = inspect.getattr_static(cls, "close", None) + if inspect.isfunction(close) and \ + not getattr(close, "_opens_data", False): + cls.close = _open_while_closing(close) + def __init__(self, flow_element): """! @@ -49,6 +195,45 @@ def __init__(self, flow_element): self.flow_element = flow_element + # Set once the FlowElement that writes the data has finished processing, + # and cleared while one that shares the instance processes. A class + # default, so data whose constructor does not call this one has it too. + _finished = False + + def __setattr__(self, name, value): + + """! + Set an attribute, unless the FlowElement that writes the data has + finished processing, when a public attribute cannot be set. Names + starting with an underscore hold the data's own state, such as + values kept once read, and are not writes. + + @raise Exception: when the FlowElement has finished processing + """ + + if self._finished and not name.startswith("_"): + self._raise_finished(name) + object.__setattr__(self, name, value) + + def _check_writable(self, name): + + """! + Raise unless the FlowElement that writes the data is processing. A + write method calls this first. + + @type name: str + @param name: what is being written, for the message + @raise Exception: when the FlowElement has finished processing + """ + + if self._finished: + self._raise_finished(name) + + def _raise_finished(self, name): + flow_element = self.__dict__.get("flow_element") + key = getattr(flow_element, "datakey", "") + raise Exception(Messages.ELEMENT_DATA_FINISHED.format(name, key)) + def get(self, key): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_dictionary.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_dictionary.py index 1becbadb..961a722c 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_dictionary.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_dictionary.py @@ -20,7 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -from .elementdata import ElementData +from .elementdata import ElementData, _contents_property class ElementDataDictionary(ElementData): @@ -30,6 +30,9 @@ class ElementDataDictionary(ElementData): """ + # A dict that refuses writes once the FlowElement has finished. + contents = _contents_property + def __init__(self, flow_element, contents): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py index ee6b8f5d..290b1af1 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py @@ -20,11 +20,17 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from .elementdata import _FlowDataDict +from .messages import Messages + # The key the web integration gives the web request itself, which the usage # sharing element reads. No element uses it as evidence, so filter() always # sets it aside, whatever the elements' filters accept. HTTP_REQUEST_KEY = "server.http-request" +# The attributes that hold Evidence values. +_VALUES = ("evidence", "unaccepted") + class Evidence: def __init__(self, flowdata): @@ -34,14 +40,33 @@ def __init__(self, flowdata): @type flowdata: FlowData @param flowdata: Parent FlowData """ + self.flowdata = flowdata self.evidence = {} # Each key in lower case to the key as first added, because keys # ignore letter case. self.keys = {} self.evidence_filtered = False - self.flowdata = flowdata self.unaccepted = {} + def __setattr__(self, name, value): + + """! + Set an attribute. The Evidence values, and the values filter() sets + aside, are kept in dicts that refuse writes once the FlowData has + finished processing, and neither can be replaced then. + + @raise Exception: when one of them is replaced once the FlowData + has finished processing + """ + + if name in _VALUES: + flowdata = self.__dict__.get("flowdata") + if name in self.__dict__ and \ + getattr(flowdata, "__dict__", {}).get("processed"): + raise Exception(Messages.FLOW_DATA_FINISHED.format("Evidence")) + value = _FlowDataDict(flowdata, "Evidence", value) + object.__setattr__(self, name, value) + def add(self, key, value): """! Add a single piece of evidence by its element and value @@ -51,8 +76,12 @@ def add(self, key, value): @type value: mixed @param value: a piece of evidence + @raise Exception: once the FlowData has finished processing """ + if self.flowdata.processed: + raise Exception(Messages.FLOW_DATA_FINISHED.format("Evidence")) + folded = str(key).lower() if folded in self.keys: key = self.keys[folded] @@ -66,9 +95,13 @@ def add_from_dict(self, evidence_dictionary): @type evidence_dictionary: dict @param evidence_dictionary: Dict of evidence + @raise Exception: once the FlowData has finished processing """ - if not type(evidence_dictionary) is dict: + if self.flowdata.processed: + raise Exception(Messages.FLOW_DATA_FINISHED.format("Evidence")) + + if not isinstance(evidence_dictionary, dict): self.flowdata.set_error("core", "Must pass valid dictionary.") for key, value in evidence_dictionary.items(): diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py index 35b0c5ad..d3a23769 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py @@ -20,12 +20,27 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from .elementdata import ElementData, _FlowDataDict from .evidence import Evidence from .flowerror import FlowError -from .messages import Messages +from .messages import Messages import traceback import sys + +def _set_finished(element_data, finished): + + # Records whether the FlowElement that writes the data has finished, which + # ElementData checks before a write. Data of another type is left alone. + if isinstance(element_data, ElementData): + object.__setattr__(element_data, "_finished", finished) + + +# The attributes that hold what a FlowData has been given, each with what +# it holds for the message when a finished FlowData is written. +_GIVEN = {"data": "ElementData", "errors": "an error", "evidence": "Evidence"} + + class FlowData: """! FlowData is created by a specific Pipeline @@ -40,6 +55,11 @@ class FlowData: request, uses it only in the code that handles that request, and closes it when the request ends. + Once processing has finished, a FlowData is read only. Adding Evidence, + ElementData or errors to it raises, whether through its methods or + through the dicts data, errors and evidence.get_all() return. Only + process() sets processed, and setting it back is not supported. + """ def __init__(self, pipeline): @@ -59,6 +79,27 @@ def __init__(self, pipeline): self.stopped = False self.evidence = Evidence(self) + def __setattr__(self, name, value): + + """! + Set an attribute. The ElementData and errors are kept in dicts that + refuse writes once processing has finished, and they and the + Evidence cannot be replaced then. + + @raise Exception: when one of them is set once processing has + finished + """ + + if name in _GIVEN: + # Read from the instance's own fields, because a missing + # attribute is treated as the key of ElementData. + if self.__dict__.get("processed"): + raise Exception(Messages.FLOW_DATA_FINISHED.format( + _GIVEN[name])) + if name != "evidence": + value = _FlowDataDict(self, _GIVEN[name], value) + object.__setattr__(self, name, value) + def process(self): """! @@ -67,37 +108,49 @@ def process(self): @rtype: FlowData @return: Returns flowdata - @raise Exception: the first error a flow element raised, or the - error saying this FlowData has already been processed, unless the - Pipeline suppresses process exceptions. Every error is in errors - either way. + @raise Exception: the first error a flow element raised, unless the + Pipeline suppresses process exceptions, and every error is in errors + either way. A second call raises the error saying this FlowData has + already been processed, as a finished FlowData takes no more errors. """ - if not self.processed: - # filter evidence before processing - try: - self.evidence.filter() - except Exception as e: - flow_error = FlowError("global", e, traceback.format_exc()) - self.set_error(flow_error) - - for flow_element in self.pipeline.flow_elements: - if self.stopped is not True: - # All errors are caught and stored in an errors array keyed by the - # flowElement that set the error - try: - flow_element.process(self) - except Exception: - flow_error = FlowError(flow_element.datakey, sys.exc_info()[1], traceback.format_exc()) - self.set_error(flow_error) - - # Set processed flag to true. flowdata can only be processed once - self.processed = True - else: - flow_error = FlowError("global", Exception(Messages.FLOW_DATA_PROCESSED), Messages.FLOW_DATA_PROCESSED) + # A finished FlowData takes no more errors, so a second call raises. + if self.processed: + raise Exception(Messages.FLOW_DATA_PROCESSED) + + # filter evidence before processing + try: + self.evidence.filter() + except Exception as e: + flow_error = FlowError("global", e, traceback.format_exc()) self.set_error(flow_error) + for flow_element in self.pipeline.flow_elements: + if self.stopped is not True: + key = str(flow_element.datakey).lower() + # A FlowElement that shares data another added may write it + # again while it processes. + _set_finished(self.data.get(key), False) + # All errors are caught and stored in an errors array keyed by the + # flowElement that set the error + try: + flow_element.process(self) + except Exception: + flow_error = FlowError(flow_element.datakey, sys.exc_info()[1], traceback.format_exc()) + self.set_error(flow_error) + finally: + _set_finished(self.data.get(key), True) + + # Data added before processing, or under another FlowElement's key, + # is finished too. + for element_data in list(self.data.values()): + _set_finished(element_data, True) + + # Set processed flag to true. flowdata can only be processed once, + # and is read only from now on. + self.processed = True + if self.errors and self.pipeline.suppress_process_exceptions is False: errored_flow_element_key = next(iter(self.errors)) flowErrorObj = self.errors[errored_flow_element_key] @@ -125,6 +178,8 @@ def close(self): # attribute it does not have as the name of a property. close = getattr(type(element_data), "close", None) if callable(close): + # Closing is not a write, and ElementData opens itself for + # its close method. close(element_data) def __enter__(self): @@ -217,9 +272,13 @@ def set_element_data(self, element_data): @type element_data: ElementData @param element_data: elementData to be added to flowdata + @raise Exception: once this FlowData has finished processing """ + if self.processed: + raise Exception(Messages.FLOW_DATA_FINISHED.format("ElementData")) + # Kept under the data key in lower case, which is how get reads it. self.data[element_data.flow_element.datakey.lower()] = element_data @@ -231,9 +290,13 @@ def set_error(self, flow_error): @type error: FlowError @param error: Flow Error + @raise Exception: once this FlowData has finished processing """ + if self.processed: + raise Exception(Messages.FLOW_DATA_FINISHED.format("an error")) + self.errors[flow_error.flow_element] = flow_error log_message = "Error occurred during processing" diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py index 5698b031..d183e5e3 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py @@ -48,10 +48,23 @@ class Messages(): # FlowData already processed. FLOW_DATA_PROCESSED = "FlowData already processed" + # A write to a FlowData that has finished processing. This takes what was + # being added as a format argument. + FLOW_DATA_FINISHED = ( + "This FlowData has finished processing, so {} cannot be added to it. " + "A FlowData is read only once processing has finished.") + + # A write to ElementData once its FlowElement has finished. This takes + # the name written and the element's data key as format arguments. + ELEMENT_DATA_FINISHED = ( + "'{}' cannot be written to the '{}' ElementData, because the " + "FlowElement that writes it has finished processing. ElementData is " + "written only while the FlowElement that provides it processes.") + # The usage sharing element is not the last Flow Element. This takes the # element's class name, its position counting from 1 and the number of # Flow Elements as format arguments. USAGE_SHARING_NOT_LAST = ( "The usage sharing element must be the last Flow Element and " "outside any parallel group. {} is Flow Element {} of {} in this " - "Pipeline.") \ No newline at end of file + "Pipeline.") diff --git a/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py new file mode 100644 index 00000000..39d7ba75 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py @@ -0,0 +1,416 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +A FlowData that has finished processing refuses Evidence, ElementData and +errors, and ElementData refuses a write once the FlowElement that writes it +has finished, as the thread safety page of the specification requires. The +last tests process every FlowElement this package defines, and its example +FlowElements, then write to each one's ElementData and expect the error. +""" + +import copy +import importlib +import importlib.util +import inspect +import json +import os +import pkgutil +import unittest + +import fiftyone_pipeline_core +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.evidence import Evidence +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.flowerror import FlowError +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +FINISHED = "finished processing" + +# FlowElements that only other FlowElements build on, which have no +# ElementData of their own to write. +BASES = {FlowElement} + +EXAMPLES = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "examples") + + +class WritingElement(FlowElement): + """Adds data and then writes it, as a FlowElement may while it + processes.""" + + def __init__(self, datakey="writer", value="first"): + super(WritingElement, self).__init__() + self.datakey = datakey + self.value = value + + def process_internal(self, flowdata): + data = flowdata.data.get(self.datakey) + if data is None: + data = ElementDataDictionary(self, {}) + flowdata.set_element_data(data) + data.written = self.value + + +class RaisingElement(FlowElement): + """Raises while it processes, so the FlowData records an error.""" + + def __init__(self, datakey="raiser"): + super(RaisingElement, self).__init__() + self.datakey = datakey + + def process_internal(self, flowdata): + raise Exception("raised while processing") + + +class AddingElement(FlowElement): + """Adds data under another FlowElement's key, and a value to its + contents while it processes.""" + + def __init__(self): + super(AddingElement, self).__init__() + self.datakey = "adder" + + def process_internal(self, flowdata): + data = ElementDataDictionary(WritingElement("other"), {}) + data.contents["item"] = "while processing" + flowdata.set_element_data(data) + + +class ClosingData(ElementDataDictionary): + """Sets a public attribute when it is closed, as a close method that + releases what the data holds may.""" + + def close(self): + self.closed = True + + +class ClosingElement(FlowElement): + """Adds data that has a close method of its own.""" + + def __init__(self): + super(ClosingElement, self).__init__() + self.datakey = "closing" + + def process_internal(self, flowdata): + flowdata.set_element_data(ClosingData(self, {})) + + +def item_store(data): + """The dictionary the data keeps its values in, or None.""" + if isinstance(data, dict): + return data + if (isinstance(getattr(type(data), "contents", None), property) + or "contents" in vars(data)): + return data.contents + return None + + +# Each way a dictionary can be written. +DICT_WRITES = ( + ("[]=", lambda d: d.__setitem__("b", 2)), + ("del", lambda d: d.__delitem__("a")), + ("update", lambda d: d.update(b=2)), + ("pop", lambda d: d.pop("a")), + ("popitem", lambda d: d.popitem()), + ("setdefault", lambda d: d.setdefault("b", 2)), + ("clear", lambda d: d.clear()), + ("|=", lambda d: d.__ior__({"b": 2})), +) + + +def process(*elements, **settings): + builder = PipelineBuilder(dict({"add_javascript_builder": False, + "use_setheader_properties": False}, + **settings)) + for element in elements: + builder.add(element) + flowdata = builder.build().create_flowdata() + flowdata.process() + return flowdata + + +def load_example(name): + """An example module, loaded from its file under its own name.""" + spec = importlib.util.spec_from_file_location( + "example_" + name, os.path.join(EXAMPLES, name + ".py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedFlowDataTests(unittest.TestCase): + + def test_evidence_cannot_be_added_once_processed(self): + flowdata = process(WritingElement()) + with self.assertRaises(Exception) as caught: + flowdata.evidence.add("query.later", "value") + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + flowdata.evidence.add_from_dict({"query.later": "value"}) + self.assertIn(FINISHED, str(caught.exception)) + self.assertIsNone(flowdata.evidence.get("query.later")) + + def test_element_data_cannot_be_added_once_processed(self): + element = WritingElement("late") + flowdata = process(WritingElement()) + with self.assertRaises(Exception) as caught: + flowdata.set_element_data(ElementDataDictionary(element, {})) + self.assertIn(FINISHED, str(caught.exception)) + self.assertNotIn("late", flowdata.data) + + def test_errors_cannot_be_added_once_processed(self): + flowdata = process(WritingElement()) + with self.assertRaises(Exception) as caught: + flowdata.set_error(FlowError("late", Exception("late"), "")) + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual({}, flowdata.errors) + + def test_a_second_process_raises_even_when_exceptions_are_suppressed(self): + flowdata = process(WritingElement(), + suppress_process_exceptions=True) + with self.assertRaises(Exception) as caught: + flowdata.process() + self.assertIn("already processed", str(caught.exception)) + self.assertEqual({}, flowdata.errors) + + def test_evidence_can_be_added_before_processing(self): + flowdata = PipelineBuilder().add(WritingElement()).build() \ + .create_flowdata() + flowdata.evidence.add("query.early", "value") + self.assertEqual("value", flowdata.evidence.get("query.early")) + + def assert_dict_writes_fail(self, finished, store): + """Each way of writing the dictionary store(finished) raises, and + leaves it as it was.""" + for name, write in DICT_WRITES: + with self.subTest(name): + flowdata = finished() + before = dict(store(flowdata)) + with self.assertRaises(Exception) as caught: + write(store(flowdata)) + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual(before, dict(store(flowdata))) + + def test_the_element_data_dictionary_cannot_be_written_once_processed( + self): + self.assert_dict_writes_fail( + lambda: process(WritingElement("a")), lambda f: f.data) + + def test_the_errors_dictionary_cannot_be_written_once_processed(self): + self.assert_dict_writes_fail( + lambda: process(RaisingElement("a"), + suppress_process_exceptions=True), + lambda f: f.errors) + + def test_the_evidence_dictionaries_cannot_be_written_once_processed( + self): + def finished(): + flowdata = PipelineBuilder().add(WritingElement()).build() \ + .create_flowdata() + flowdata.evidence.add("a", "value") + flowdata.process() + return flowdata + + for store in (lambda f: f.evidence.get_all(), + lambda f: f.evidence.evidence, + lambda f: f.evidence.get_unaccepted()): + self.assert_dict_writes_fail(finished, store) + + def test_the_dictionaries_cannot_be_replaced_once_processed(self): + flowdata = process(WritingElement()) + evidence = flowdata.evidence + for owner, name, value in ( + (flowdata, "data", {}), + (flowdata, "errors", {}), + (flowdata, "evidence", Evidence(flowdata)), + (evidence, "evidence", {}), + (evidence, "unaccepted", {})): + with self.subTest(name): + before = getattr(owner, name) + with self.assertRaises(Exception) as caught: + setattr(owner, name, value) + self.assertIn(FINISHED, str(caught.exception)) + self.assertIs(before, getattr(owner, name)) + + def test_data_given_before_processing_is_read_only_after(self): + flowdata = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(WritingElement()).build().create_flowdata() + flowdata.data = {"given": "before processing"} + flowdata.errors = {} + flowdata.data["added"] = "before processing" + flowdata.process() + self.assertEqual("before processing", flowdata.get("given")) + self.assertEqual("before processing", flowdata.get("added")) + for store in (flowdata.data, flowdata.errors): + with self.assertRaises(Exception) as caught: + store["later"] = "after processing" + self.assertIn(FINISHED, str(caught.exception)) + + +class FinishedElementDataTests(unittest.TestCase): + + def test_data_written_while_processing_cannot_be_written_after(self): + flowdata = process(WritingElement()) + data = flowdata.get("writer") + self.assertEqual("first", data.written) + with self.assertRaises(Exception) as caught: + data.written = "after" + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual("first", data.written) + + def test_an_element_sharing_the_data_writes_it_while_it_processes(self): + flowdata = process(WritingElement("shared", "first"), + WritingElement("shared", "second")) + data = flowdata.get("shared") + self.assertEqual("second", data.written) + with self.assertRaises(Exception): + data.written = "after" + + def test_the_data_keeps_its_own_state_once_finished(self): + data = process(WritingElement()).get("writer") + data._kept = "a value kept once read" + self.assertEqual("a value kept once read", data._kept) + + def test_the_contents_cannot_be_written_once_finished(self): + for name, write in DICT_WRITES: + with self.subTest(name): + data = process(WritingElement()).get("writer") + dict.__setitem__(data.contents, "a", 1) + for contents in (data.contents, data.as_dictionary()): + with self.assertRaises(Exception) as caught: + write(contents) + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual({"a": 1}, dict(data.contents)) + + def test_the_contents_cannot_be_replaced_once_finished(self): + data = process(WritingElement()).get("writer") + with self.assertRaises(Exception) as caught: + data.contents = {"b": 2} + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual({}, dict(data.contents)) + + def test_the_contents_are_a_dict_that_is_written_while_processing(self): + flowdata = process(AddingElement()) + contents = flowdata.get("other").contents + self.assertIsInstance(contents, dict) + self.assertEqual({"item": "while processing"}, contents) + self.assertEqual('{"item": "while processing"}', + json.dumps(contents)) + self.assertIs(dict, type(copy.deepcopy(contents))) + self.assertIs(dict, type(contents.copy())) + + def test_data_under_another_key_is_finished_too(self): + flowdata = process(AddingElement()) + data = flowdata.get("other") + with self.assertRaises(Exception) as caught: + data.written = "after" + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + data.contents["item"] = "after" + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual("while processing", data.contents["item"]) + + def test_data_added_before_processing_is_finished_with_it(self): + flowdata = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(WritingElement()).build().create_flowdata() + data = ElementDataDictionary(WritingElement("early"), {}) + flowdata.set_element_data(data) + data.written = "before processing" + flowdata.process() + with self.assertRaises(Exception) as caught: + data.written = "after" + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual("before processing", data.written) + + def test_closing_the_flow_data_is_not_a_write(self): + flowdata = process(ClosingElement()) + flowdata.close() + self.assertTrue(flowdata.get("closing").closed) + + def test_closing_the_data_itself_is_not_a_write(self): + data = process(ClosingElement()).get("closing") + data.close() + self.assertTrue(data.closed) + with self.assertRaises(Exception) as caught: + data.closed = False + self.assertIn(FINISHED, str(caught.exception)) + self.assertTrue(data.closed) + + +class EveryFlowElementTests(unittest.TestCase): + + def assert_writes_fail(self, flowdata, key): + data = flowdata.get(key) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + contents = item_store(data) + if contents is not None: + with self.assertRaises(Exception) as caught: + contents["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) + + def test_the_package_flow_elements_refuse_writes_once_finished(self): + flowdata = PipelineBuilder().build().create_flowdata() + flowdata.evidence.add("header.user-agent", "a user agent") + flowdata.process() + processed = set() + for element in flowdata.pipeline.flow_elements: + self.assert_writes_fail(flowdata, element.datakey) + processed.add(type(element)) + self.assertEqual( + defined_flow_elements(fiftyone_pipeline_core) - BASES, processed) + + def test_the_example_flow_elements_refuse_writes_once_finished(self): + for name, element in ( + ("simple_flow_element", "SimpleFlowElement"), + ("client_side_evidence_custom_flow_element", + "AstrologyFlowElement")): + module = load_example(name) + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + flowdata = builder.add(getattr(module, element)()).build() \ + .create_flowdata() + flowdata.evidence.add("query.dateOfBirth", "1992-12-18") + flowdata.process() + self.assert_writes_fail(flowdata, "astrology") + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_setheader.py b/pipeline/fiftyone_pipeline_core/tests/test_setheader.py index 9f96df6a..67311854 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_setheader.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_setheader.py @@ -57,7 +57,9 @@ class SetHeaderUACHTests(unittest.TestCase): def test_get_response_header_value(self, device, expected_value): testPipeline = TestPipeline() setHeaderElement = SetHeaderElement() - flowdata = testPipeline.flowdata + # A FlowData that has not been processed, because a finished one + # takes no more ElementData. + flowdata = testPipeline.pipeline.create_flowdata() flowdata.data = device actual_value = setHeaderElement.get_response_header_value(flowdata, setHeader_properties_dict) self.assertEqual(expected_value, actual_value[Constants.ACCEPTCH_HEADER]) @@ -80,7 +82,9 @@ def test_set_response_header(self, data, expected_value): response = session.get('mock://test.com/3') testPipeline = TestPipeline() - flowdata = testPipeline.flowdata + # A FlowData that has not been processed, because a finished one + # takes no more ElementData. + flowdata = testPipeline.pipeline.create_flowdata() flowdata.data = data response = set_response_header(flowdata, response) diff --git a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_data.py b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_data.py index dcca50e6..96e9c9b7 100644 --- a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_data.py +++ b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_data.py @@ -36,5 +36,11 @@ def __init__(self, flow_element): super().__init__(flow_element, {}) def set_value(self, name, value): - """! Writes the value of one property. """ + """! + Writes the value of one property. + + @raise Exception: once every element that writes the data has + finished processing + """ + self._check_writable(name) self.contents[name.lower()] = value diff --git a/pipeline/fiftyone_pipeline_derivedproperty/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_finished_writes.py new file mode 100644 index 00000000..d02452c1 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_finished_writes.py @@ -0,0 +1,105 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines, and the one its example defines, is +processed, then a write to its ElementData is expected to fail, as the +thread safety page of the specification requires. The derived property +elements of a pipeline share one ElementData, which each writes while it +processes and none writes once they have all finished. +""" + +import importlib +import importlib.util +import inspect +import os +import pkgutil + +import pytest + +import fiftyone_pipeline_derivedproperty +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_derivedproperty import DerivedPropertyElementBuilder +from fiftyone_pipeline_derivedproperty.derived_property_element import \ + DerivedPropertyElement + +from .helpers import SourceElement, band_script, pipeline_of, processed + +FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + +EXAMPLES = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "examples") + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +def derived(name): + return (DerivedPropertyElementBuilder() + .add_script(name, band_script(name, "src.N")).build()) + + +def test_every_derived_element_writes_only_while_it_processes(): + flowdata = processed(pipeline_of( + [SourceElement("src", {"N": 12}), derived("Demo"), derived("Other")])) + data = flowdata.derived + assert data.get("Demo").value() == "High" + assert data.get("Other").value() == "High" + with pytest.raises(Exception, match=FINISHED): + data.set_value("Demo", "written after processing") + with pytest.raises(Exception, match=FINISHED): + data.written_after_processing = True + with pytest.raises(Exception, match=FINISHED): + item_store(data)["written_after_processing"] = True + assert data.get("Demo").value() == "High" + assert defined_flow_elements(fiftyone_pipeline_derivedproperty) == \ + {DerivedPropertyElement} + + +def test_the_example_flow_element_refuses_writes_once_finished(): + spec = importlib.util.spec_from_file_location( + "example_screen_element", os.path.join(EXAMPLES, "screen_element.py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + flowdata = processed(pipeline_of([module.ScreenElement()])) + with pytest.raises(Exception, match=FINISHED): + flowdata.get("screen").written_after_processing = True + with pytest.raises(Exception, match=FINISHED): + item_store(flowdata.get("screen"))["written_after_processing"] = True diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata_dictionary.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata_dictionary.py index b259bdcb..88a6dd14 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata_dictionary.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata_dictionary.py @@ -20,6 +20,8 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from fiftyone_pipeline_core.elementdata import _contents_property + from .aspectdata import AspectData class AspectDataDictionary(AspectData): @@ -30,6 +32,9 @@ class AspectDataDictionary(AspectData): """ + # A dict that refuses writes once the FlowElement has finished. + contents = _contents_property + def __init__(self, flow_element, contents): """! diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_engines/tests/test_finished_writes.py new file mode 100644 index 00000000..fdb45ade --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines/tests/test_finished_writes.py @@ -0,0 +1,97 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +The engines package defines only the Engine that other FlowElements build +on. Its example engine is processed, then a write to its ElementData is +expected to fail, as the thread safety page of the specification requires. +""" + +import importlib +import importlib.util +import inspect +import os +import pkgutil +import unittest + +import fiftyone_pipeline_engines +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_engines.engine import Engine + +FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + +EXAMPLES = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "examples") + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_the_package_defines_only_the_engine_others_build_on(self): + self.assertEqual( + {Engine}, defined_flow_elements(fiftyone_pipeline_engines)) + + def test_the_example_engine_refuses_writes_once_finished(self): + spec = importlib.util.spec_from_file_location( + "example_simple_onpremise_engine", + os.path.join(EXAMPLES, "simple_onpremise_engine.py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + engine = module.SimpleOnPremiseEngine(module.DATA_FILE) + pipeline = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(engine).build() + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + flowdata.evidence.add(module.DATE_OF_BIRTH, "1992-12-18") + flowdata.process() + with self.assertRaises(Exception) as caught: + flowdata.get("astrology").written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(flowdata.get("astrology"))["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_finished_writes.py new file mode 100644 index 00000000..6c4b6400 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_finished_writes.py @@ -0,0 +1,80 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, as the thread safety +page of the specification requires. The usage sharing element adds no +ElementData, so a finished FlowData holds none for it, and the FlowData +refuses one being added. +""" + +import importlib +import inspect +import pkgutil +import unittest + +import fiftyone_pipeline_engines_fiftyone +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_engines_fiftyone.share_usage import ShareUsage + +FINISHED = "finished processing" + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + # Nothing listens on the discard port, and one record is far short + # of a package, so nothing is sent. + usage = ShareUsage(requested_package_size=1000, + endpoint="http://127.0.0.1:9") + pipeline = PipelineBuilder().add(usage).build() + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "a user agent") + flowdata.process() + self.assertNotIn(usage.datakey, flowdata.data) + with self.assertRaises(Exception) as caught: + flowdata.set_element_data(ElementDataDictionary(usage, {})) + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual( + {ShareUsage}, + defined_flow_elements(fiftyone_pipeline_engines_fiftyone)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_translation/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_translation/tests/test_finished_writes.py new file mode 100644 index 00000000..ac741857 --- /dev/null +++ b/pipeline/fiftyone_pipeline_translation/tests/test_finished_writes.py @@ -0,0 +1,99 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. +""" + +import importlib +import inspect +import pkgutil +import unittest + +import fiftyone_pipeline_translation +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_translation.translation_engine_base import ( + TranslationEngineBase) + +FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class SourceElement(FlowElement): + """Gives fixed names to translate.""" + + def __init__(self): + super(SourceElement, self).__init__() + self.datakey = "source" + + def process_internal(self, flowdata): + flowdata.set_element_data( + ElementDataDictionary(self, {"names": ["Germany"]})) + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + engine = TranslationEngineBase( + "source", [("Names", "NamesTranslated")], + {"countries.fr_FR.yml": "Germany: Allemagne\n"}, "fr_FR") + flowdata = PipelineBuilder().add(SourceElement()).add(engine) \ + .build().create_flowdata() + flowdata.process() + data = flowdata.get(engine.datakey) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual( + {TranslationEngineBase}, + defined_flow_elements(fiftyone_pipeline_translation)) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file