diff --git a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py index a19b7da2..b6e6fe5e 100644 --- a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py @@ -23,7 +23,7 @@ from __future__ import absolute_import from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine from fiftyone_pipeline_cloudrequestengine.clouddata import \ - CloudMissingPropertyService + OnPremiseMissingPropertyService from fiftyone_pipeline_engines.aspectdata import AspectData from fiftyone_pipeline_engines.aspectdata_dictionary import AspectDataDictionary @@ -91,7 +91,7 @@ def __init__(self, flow_element, contents): # The device data's service, so a name that is not a property is # reported in the same way for both. - self.missing_property_service = CloudMissingPropertyService() + self.missing_property_service = OnPremiseMissingPropertyService() def _matching_key(self, key): """! diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py index 702e5c04..7d769b84 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py @@ -30,8 +30,6 @@ from fiftyone_devicedetection_cloud.devicedetection_cloud import DeviceDetectionCloud -from fiftyone_pipeline_engines.missingproperty_service import CONFIGURATOR_URL - from .live_cloud import find_resource_key, no_resource_key pytestmark = pytest.mark.live @@ -158,16 +156,10 @@ def test_missing_property_service_not_found_anywhere(self): fd.device.get("notpresent") result = str(raised.exception) - properties = pipeline.get_element("device").get_properties() - self.assertIn( - "Property 'notpresent' not found in data for element 'device'. " - "This is because your resource key does not include access to " - "this property. The properties included for your key under " - "'device' are ", result) - self.assertIn( - ",".join(meta.get("name", name) - for name, meta in properties.items()), result) - self.assertIn(CONFIGURATOR_URL, result) + self.assertIn("notpresent", result) + self.assertIn("device", result) + self.assertIn(', '.join(pipeline.get_element("device").get_properties().keys()), result) + self.assertIn("https://51degrees.com/documentation/_services__cloud__resource_keys.html", result) def test_cloud_request_origin(self): """! diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index 05f4cc0f..7d7891fa 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -25,8 +25,7 @@ from fiftyone_pipeline_engines.aspectdata import AspectData from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue -from fiftyone_pipeline_engines.missingproperty_service import \ - OnPremiseMissingPropertyService +from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService from .constants import RESULTS_CLOSED @@ -34,6 +33,22 @@ # fiftyoneDegreesHashMatchMethod values, which getMethod returns. MATCH_METHODS = ("NONE", "PERFORMANCE", "COMBINED", "PREDICTIVE") +class OnPremiseMissingPropertyService(MissingPropertyService): + + def check(self, key, flow_element): + + if key in flow_element.properties: + datafiles = flow_element.properties[key]["datafiles"] + if(not flow_element.engine.getProduct() in datafiles): + raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. This is because your datafile does not contain the property. The property is available in' + str(datafiles)) + # The data file holds the property, so it is missing because + # it is not available. A missing property always raises. + raise Exception( + "Property '" + key + "' not found in data for element '" + + flow_element.datakey + "'. This is because the property " + "has been excluded when configuring the engine.") + else: + raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. Please check that the element and property names are correct.') class SwigData(AspectData): diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py index aa6222ca..f8151eaf 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py @@ -261,9 +261,7 @@ def test_missing_property_service_not_found_anywhere(self): result = str(e) self.assertEqual( - "Property 'notpresent' not found in data for element 'device'. " - "The reason for this is unknown. Please check that the aspect and " - "property name are correct.", result) + result, "Property notpresent not found in data for element device. Please check that the element and property names are correct.") def test_missing_property_service_not_found_in_current(self): """! @@ -288,20 +286,15 @@ def test_missing_property_service_not_found_in_current(self): fd.process() - # The name is found however it is spelled, and the message gives it - # as the caller did. - for name in ("mock", "Mock"): - with self.assertRaises(Exception) as raised: - fd.device.get(name) - self.assertEqual( - "Property '" + name + "' not found in data for element " - "'device'. This is because your license or data file does not " - "include this property. It is available with the Enterprise " - "license/data for the DeviceDetectionOnPremise. To upgrade, " - "see " - "https://51degrees.com/pricing?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-pricing" - ".", - str(raised.exception)) + result = "" + + try: + fd.device.get("mock") + except Exception as e: + result = str(e) + + self.assertEqual( + result, "Property mock not found in data for element device. This is because your datafile does not contain the property. The property is available in['Enterprise']") def test_excluded_property(self): """! @@ -329,9 +322,7 @@ def test_excluded_property(self): result = str(e) self.assertEqual( - "Property 'screenpixelsheight' not found in data for element " - "'device'. This is because the property has been excluded when " - "configuring the engine.", result) + result, "Property screenpixelsheight was excluded from device") def test_evidencekey_filter_contains_user_agent(self): """! diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_missing_property_raises.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_missing_property_raises.py new file mode 100644 index 00000000..2fe00eaa --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_missing_property_raises.py @@ -0,0 +1,56 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""A property the engine lists, in a data file tier that holds it, raises +when the data has no value for it, as every missing property does.""" + +import unittest +from types import SimpleNamespace + +from fiftyone_devicedetection_onpremise.swig_data import \ + OnPremiseMissingPropertyService + + +class StandInEngine: + """An engine whose data file is Lite, listing one property that the + Lite file holds.""" + + datakey = "device" + properties = {"ismobile": {"datafiles": ["Lite", "Enterprise"]}} + engine = SimpleNamespace(getProduct=lambda: "Lite") + + +class MissingPropertyRaisesTests(unittest.TestCase): + + def test_a_listed_property_of_this_tier_raises(self): + with self.assertRaises(Exception) as raised: + OnPremiseMissingPropertyService().check( + "ismobile", StandInEngine()) + + self.assertEqual( + "Property 'ismobile' not found in data for element 'device'. " + "This is because the property has been excluded when " + "configuring the engine.", str(raised.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index 2070a22b..85fa7094 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -27,8 +27,7 @@ from fiftyone_pipeline_engines.aspectdata import AspectData from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue -from fiftyone_pipeline_engines.missingproperty_service import \ - OnPremiseMissingPropertyService +from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService from .constants import RESULTS_CLOSED @@ -43,6 +42,24 @@ COORDINATE_PROPERTIES = ("latitude", "longitude") +class OnPremiseMissingPropertyService(MissingPropertyService): + + def check(self, key, flow_element): + + if key in flow_element.properties: + datafiles = flow_element.properties[key]["datafiles"] + if(not flow_element.engine.getProduct() in datafiles): + raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. This is because your datafile does not contain the property. The property is available in' + str(datafiles)) + # The data file holds the property, so it is missing because + # it is not available. A missing property always raises. + raise Exception( + "Property '" + key + "' not found in data for element '" + + flow_element.datakey + "'. This is because the property " + "has been excluded when configuring the engine.") + else: + raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. Please check that the element and property names are correct.') + + class SwigData(AspectData): """! diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_missing_property_raises.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_missing_property_raises.py new file mode 100644 index 00000000..f8e282e7 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_missing_property_raises.py @@ -0,0 +1,56 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""A property the engine lists, in a data file tier that holds it, raises +when the data has no value for it, as every missing property does.""" + +import unittest +from types import SimpleNamespace + +from fiftyone_ipintelligence_onpremise.swig_data import \ + OnPremiseMissingPropertyService + + +class StandInEngine: + """An engine whose data file is Lite, listing one property that the + Lite file holds.""" + + datakey = "ip" + properties = {"registeredname": {"datafiles": ["Lite", "Enterprise"]}} + engine = SimpleNamespace(getProduct=lambda: "Lite") + + +class MissingPropertyRaisesTests(unittest.TestCase): + + def test_a_listed_property_of_this_tier_raises(self): + with self.assertRaises(Exception) as raised: + OnPremiseMissingPropertyService().check( + "registeredname", StandInEngine()) + + self.assertEqual( + "Property 'registeredname' not found in data for element 'ip'. " + "This is because the property has been excluded when " + "configuring the engine.", str(raised.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py index e45f67fd..4f4f8c88 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py @@ -22,48 +22,27 @@ from fiftyone_pipeline_engines.aspectdata_dictionary import AspectDataDictionary -from fiftyone_pipeline_engines.missingproperty_service import \ - MissingPropertyReason, MissingPropertyService - - -class CloudMissingPropertyService(MissingPropertyService): - - """! - The missing property service of a Cloud Aspect Engine. A property its - metadata does not list is one the key in use does not include, or, - while the engine's property list is not known, one whose request for - that list failed. - """ - - def engine_reason(self, flow_element): - - """! - The reason for a property the metadata does not list. - - @type flow_element: CloudEngine - @param flow_element: the engine - @rtype: tuple - @return: the reason, and the property names the key includes - """ - - properties_known = getattr(flow_element, "properties_known", None) - if properties_known is not None and not properties_known(): - # The Cloud Request Engine asks for the property list when it is - # built, so a list that is still not known is one whose request - # failed. - return MissingPropertyReason.CLOUD_REQUEST_FAILED, () - properties = self.properties(flow_element) - if not properties: - return ( - MissingPropertyReason.PRODUCT_NOT_ACCESSIBLE_WITH_RESOURCE_KEY, - ()) - return ( - MissingPropertyReason.PROPERTY_NOT_ACCESSIBLE_WITH_RESOURCE_KEY, - [meta.get("name", name) for name, meta in properties.items()]) - - -# The name this service had before, kept for code that imports it. -OnPremiseMissingPropertyService = CloudMissingPropertyService +from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService + +class OnPremiseMissingPropertyService(MissingPropertyService): + + def check(self, key, flow_element): + + # The properties listed here are the ones the key carries, which is + # not the same as the ones one answer holds: a request made with a + # licence key carries only the properties it asked for, so a + # property can be listed and still be absent. The message names + # both causes, so the reader checks the right one. + raise Exception( + "Property " + key + " not found in data for element " + + flow_element.datakey + ". Either the key in use does not " + "include access to this property, or the request was made " + "with a licence key and the property was not among the " + "requested_properties. Properties the key carries under " + + flow_element.datakey + " are " + + ", ".join(list(flow_element.get_properties().keys())) + + ". For more details on resource keys, see our explainer: " + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-clouddata.py&utm_term=property-not-found") class CloudData(AspectDataDictionary): @@ -83,4 +62,4 @@ def __init__(self, flow_element, contents): super(CloudData, self).__init__(flow_element, contents) - self.missing_property_service = CloudMissingPropertyService() + self.missing_property_service = OnPremiseMissingPropertyService() diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py index 1352adde..248ec92e 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py @@ -26,12 +26,14 @@ from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue from fiftyone_pipeline_core.jsonbundler import WKT_TYPES -from .clouddata import CloudData, CloudMissingPropertyService +from .clouddata import CloudData from .constants import Constants import json import threading +from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService + # Why a pipeline with a second CloudRequestEngine is refused. _ANOTHER_REQUEST_ENGINE = ( "This cloud engine is already in a pipeline with another Cloud Request " @@ -39,9 +41,11 @@ "pipeline with a different one is refused. Build another cloud engine " "for this pipeline, or use the same Cloud Request Engine in both.") -# The name the cloud missing property service had here, kept for code that -# imports it. -OnPremiseMissingPropertyService = CloudMissingPropertyService +class OnPremiseMissingPropertyService(MissingPropertyService): + + def check(self, key, flow_element): + + raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. Please check that the element and property names are correct.') class CloudEngine(Engine): @@ -128,20 +132,6 @@ def get_properties(self): self._load_properties(self._log) return super(CloudEngine, self).get_properties() - def properties_known(self): - - """! - Whether this engine has its properties from the CloudRequestEngine, - which it has once that engine's discovery has completed. The - properties are looked for first, as get_properties does. - - @rtype: bool - @return: True once the properties are known - """ - - self._load_properties(self._log) - return self._properties_final - def _load_properties(self, log): """! diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_property.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_property.py deleted file mode 100644 index 92117abc..00000000 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_property.py +++ /dev/null @@ -1,133 +0,0 @@ -# ********************************************************************* -# This Original Work is copyright of 51 Degrees Mobile Experts Limited. -# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, -# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. -# -# This Original Work is licensed under the European Union Public Licence -# (EUPL) v.1.2 and is subject to its terms as set out below. -# -# If a copy of the EUPL was not distributed with this file, You can obtain -# one at https://opensource.org/licenses/EUPL-1.2. -# -# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be -# amended by the European Commission) shall be deemed incompatible for -# the purposes of the Work and the provisions of the compatibility -# clause in Article 5 of the EUPL shall not apply. -# -# If using the Work as, or as part of, a network application, by -# including the attribution notice(s) required under Article 5 of the EUPL -# in the end user terms of the application under an appropriate heading, -# such notice(s) shall fulfill the requirements of that article. -# ********************************************************************* - -""" -The reason a Cloud Aspect Engine gives for a missing property, decided in -the order the properties specification sets out, with the message of the -missing properties table for that reason. -""" - -import json -import unittest - -from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine -from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary -from fiftyone_pipeline_core.flowelement import FlowElement -from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder - -# The configurator the Resource Key rows link to, with this repository's -# campaign parameters. -CONFIGURATOR = "https://configure.51degrees.com/?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-configurator" - - -class FakeCloudRequestEngine(FlowElement): - """Stands in for the Cloud Request Engine of a key that carries only the - device product, with two of its properties.""" - - def __init__(self): - super(FakeCloudRequestEngine, self).__init__() - self.datakey = "cloud" - self.flow_element_properties = {"device": { - "ismobile": {"name": "IsMobile", "type": "Boolean"}, - "hardwarevendor": {"name": "HardwareVendor", "type": "String"}}} - - def process_internal(self, flowdata): - flowdata.set_element_data(ElementDataDictionary( - self, {"cloud": json.dumps({"device": {"ismobile": True}})})) - - -class UndiscoveredCloudRequestEngine(FakeCloudRequestEngine): - """Stands in for a Cloud Request Engine whose request for the property - list failed when it was built, so its properties are not known.""" - - def known_flow_element_properties(self): - return None - - -def build(request_engine, datakey, restricted=None): - engine = CloudEngine() - engine.datakey = datakey - if restricted is not None: - engine.set_restricted_properties(restricted) - pipeline = PipelineBuilder({ - "add_javascript_builder": False, - "use_setheader_properties": False}).add( - request_engine).add(engine).build() - flowdata = pipeline.create_flowdata() - flowdata.process() - return pipeline, flowdata - - -class CloudReasonTests(unittest.TestCase): - - def message(self, request_engine, datakey, name, restricted=None): - pipeline, flowdata = build(request_engine, datakey, restricted) - self.addCleanup(pipeline.close) - with self.assertRaises(Exception) as raised: - flowdata.get(datakey).get(name) - return str(raised.exception) - - def test_a_product_the_key_does_not_include(self): - self.assertEqual( - "Property 'country' not found in data for element 'location'. " - "This is because your resource key does not include access to " - "any properties under 'location'. Check or create a resource key " - "with the properties you need in the configurator at " - + CONFIGURATOR + ".", - self.message(FakeCloudRequestEngine(), "location", "country")) - - def test_a_property_the_key_does_not_include(self): - self.assertEqual( - "Property 'browsername' not found in data for element 'device'. " - "This is because your resource key does not include access to " - "this property. The properties included for your key under " - "'device' are IsMobile,HardwareVendor. Check or change your key " - "in the configurator at " + CONFIGURATOR + ".", - self.message(FakeCloudRequestEngine(), "device", "browsername")) - - def test_a_listed_property_not_in_the_answer_was_excluded(self): - self.assertEqual( - "Property 'HardwareVendor' not found in data for element " - "'device'. This is because the property has been excluded when " - "configuring the engine.", - self.message( - FakeCloudRequestEngine(), "device", "HardwareVendor")) - - def test_a_property_left_out_of_the_restricted_list_was_excluded(self): - self.assertIn( - "This is because the property has been excluded when " - "configuring the engine.", - self.message( - FakeCloudRequestEngine(), "device", "hardwarevendor", - restricted=["IsMobile"])) - - def test_a_failed_request_for_the_property_list(self): - self.assertEqual( - "Property 'browsername' not found in data for element 'device'. " - "This is because an upstream cloud request failed, so no data " - "was available to populate the property.", - self.message( - UndiscoveredCloudRequestEngine(), "device", "browsername")) - - -if __name__ == "__main__": - unittest.main() diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py index 1f42abc6..72b5a11c 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py @@ -595,12 +595,13 @@ def test_no_report_when_every_property_came_back(self): str(warning.message) for warning in caught if "not covered" in str(warning.message)]) - def test_missing_property_message_names_the_properties_carried(self): + def test_missing_property_message_names_both_causes(self): """ - With a licence key, a property the key does not carry gives the - missing properties table's message for it, which lists the - properties the key carries under the element it looked in rather - than assuming device. + With a licence key the accessible properties list everything the + licence entitles, so a property can be listed as carried and still + be absent from an answer because it was not requested. The message + for a property that is not in the data says so, and names the + element it looked in rather than assuming device. """ engine = CloudRequestEngine({ @@ -620,8 +621,7 @@ def test_missing_property_message_names_the_properties_carried(self): data.device.get("other") message = str(context.exception) - self.assertIn( - "Property 'other' not found in data for element 'device'. This " - "is because your resource key does not include access to this " - "property. The properties included for your key under 'device' " - "are value.", message) + self.assertIn("Property other not found", message) + self.assertIn("requested_properties", message) + self.assertIn("under device are value", message) + self.assertNotIn("your resource key", message) diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py index 28faf87e..fa02ff16 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py @@ -61,17 +61,16 @@ def get(self, key): @rtype mixed @return The value stored under the chosen key @raise Exception: where the property was left out of the engine's - restricted properties, or the data has no value for it, with the - message the missing property service gives for the reason + restricted properties, saying so, or where the data has no value + for it, with the message the missing property service gives """ if hasattr(self.flow_element, "restricted_property_keys"): if key.lower() not in self.flow_element.restricted_property_keys: - # The service gives the reason, as the property was - # excluded or is in a data tier the data file lacks. - return self.missing_property_service.check( - key, self.flow_element) + raise Exception( + "Property " + key + " was excluded from " + + self.flow_element.datakey) try: if self.get_internal(key) == None: diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/missingproperty_service.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/missingproperty_service.py index 35c4a7c6..c99faa00 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/missingproperty_service.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/missingproperty_service.py @@ -20,104 +20,13 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -from enum import Enum - - -class MissingPropertyReason(Enum): - """! - Why a property an engine defines is missing from its data, named as the - .NET packages name each reason. - """ - - DATA_FILE_UPGRADE_REQUIRED = "DataFileUpgradeRequired" - PROPERTY_EXCLUDED_FROM_ENGINE_CONFIGURATION = \ - "PropertyExcludedFromEngineConfiguration" - PRODUCT_NOT_ACCESSIBLE_WITH_RESOURCE_KEY = \ - "ProductNotAccessibleWithResourceKey" - PROPERTY_NOT_ACCESSIBLE_WITH_RESOURCE_KEY = \ - "PropertyNotAccessibleWithResourceKey" - CLOUD_REQUEST_FAILED = "CloudRequestFailed" - UNKNOWN = "Unknown" - - -# The pages the messages link to, with this repository's campaign -# parameters. -PRICING_URL = "https://51degrees.com/pricing?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-pricing" -CONFIGURATOR_URL = "https://configure.51degrees.com/?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-configurator" - -# Lists in a message are joined with a comma and no space. -LIST_SEPARATOR = "," - - -def missing_property_message( - reason, name, element, tiers=(), engine_type="", properties=()): - """! - The message of the missing properties table for a reason. - - @type reason: MissingPropertyReason - @param reason: why the property is missing - @type name: str - @param name: the property, as the caller named it - @type element: str - @param element: the data key of the element asked - @type tiers: list - @param tiers: the data tiers that hold the property, for - DATA_FILE_UPGRADE_REQUIRED, where an empty list gives the row for - tiers that are not known - @type engine_type: str - @param engine_type: the engine's type name, for - DATA_FILE_UPGRADE_REQUIRED - @type properties: list - @param properties: the property names the key includes, for - PROPERTY_NOT_ACCESSIBLE_WITH_RESOURCE_KEY - @rtype: str - @return: the message - """ - - text = ("Property '" + name + "' not found in data for element '" + - element + "'. ") - if reason is MissingPropertyReason.DATA_FILE_UPGRADE_REQUIRED: - text += ("This is because your license or data file does not " - "include this property. ") - if tiers: - return text + ( - "It is available with the " + LIST_SEPARATOR.join(tiers) + - " license/data for the " + engine_type + - ". To upgrade, see " + PRICING_URL + ".") - return text + ( - "To see which licenses include it, see " + PRICING_URL + ".") - if reason is MissingPropertyReason.PROPERTY_EXCLUDED_FROM_ENGINE_CONFIGURATION: - return text + ("This is because the property has been excluded " - "when configuring the engine.") - if reason is MissingPropertyReason.PRODUCT_NOT_ACCESSIBLE_WITH_RESOURCE_KEY: - return text + ( - "This is because your resource key does not include access to " - "any properties under '" + element + "'. Check or create a " - "resource key with the properties you need in the configurator " - "at " + CONFIGURATOR_URL + ".") - if reason is MissingPropertyReason.PROPERTY_NOT_ACCESSIBLE_WITH_RESOURCE_KEY: - return text + ( - "This is because your resource key does not include access to " - "this property. The properties included for your key under '" + - element + "' are " + LIST_SEPARATOR.join(properties) + - ". Check or change your key in the configurator at " + - CONFIGURATOR_URL + ".") - if reason is MissingPropertyReason.CLOUD_REQUEST_FAILED: - return text + ( - "This is because an upstream cloud request failed, so no data " - "was available to populate the property.") - return text + ("The reason for this is unknown. Please check that the " - "aspect and property name are correct.") - class MissingPropertyService(): """! - Raises the error for a property an engine's data does not hold, giving - the reason in the order the properties specification sets out, with - the missing properties table's message for it. Property names are - compared ignoring case. Engines of a kind override needs_upgrade and - engine_reason to add their own steps. + A missing property service runs when a property is not available in the + aspectData. It can be extended to provide a specific message for why the property + is not available """ def check(self, key, flow_element): @@ -129,121 +38,7 @@ def check(self, key, flow_element): @param key: the property asked for @type flow_element: FlowElement @param flow_element: the element whose data was asked - @raise Exception: always, with the message for the reason the - property is missing - """ - - raise Exception(self.message(key, flow_element)) - - def message(self, key, flow_element): - - """! - The message for the reason a property is missing. - - @type key: string - @param key: the property asked for - @type flow_element: FlowElement - @param flow_element: the element whose data was asked - @rtype: str - @return: the message - """ - - name = key.lower() - listed = None - for property_name, meta in self.properties(flow_element).items(): - if property_name.lower() == name: - listed = meta - break - restricted = getattr(flow_element, "restricted_property_keys", None) - - # A property the metadata lists, or one the restricted list leaves - # out, is in a data tier the data file does not hold, or was - # excluded. - if listed is not None or ( - restricted is not None and name not in restricted): - tiers = list((listed or {}).get("datafiles") or ()) - if listed is not None and self.needs_upgrade(tiers, flow_element): - return missing_property_message( - MissingPropertyReason.DATA_FILE_UPGRADE_REQUIRED, key, - flow_element.datakey, tiers=tiers, - engine_type=type(flow_element).__name__) - return missing_property_message( - MissingPropertyReason.PROPERTY_EXCLUDED_FROM_ENGINE_CONFIGURATION, - key, flow_element.datakey) - - reason, properties = self.engine_reason(flow_element) - return missing_property_message( - reason, key, flow_element.datakey, properties=properties) - - def properties(self, flow_element): - - """! - The engine's property metadata, by name. - - @type flow_element: FlowElement - @param flow_element: the engine - @rtype: dict - @return: the metadata of each property, by name - """ - - get_properties = getattr(flow_element, "get_properties", None) - if get_properties is not None: - return get_properties() or {} - return getattr(flow_element, "properties", None) or {} - - def needs_upgrade(self, tiers, flow_element): - - """! - Whether a property the metadata lists is missing because the - engine's data file is not of one of the property's tiers. An engine - with no data file never needs an upgrade. - - @type tiers: list - @param tiers: the data tiers that hold the property - @type flow_element: FlowElement - @param flow_element: the engine - @rtype: bool - @return: True where an upgrade is needed - """ - - return False - - def engine_reason(self, flow_element): - - """! - The reason for a property the metadata does not list, which a kind - of engine with a step of its own overrides. - - @type flow_element: FlowElement - @param flow_element: the engine - @rtype: tuple - @return: the reason, and the property names for a message that - lists them - """ - - return MissingPropertyReason.UNKNOWN, () - - -class OnPremiseMissingPropertyService(MissingPropertyService): - - """! - The missing property service of an on-premise engine, whose data file - tier decides whether a property the metadata lists needs an upgrade. - The engine's native engine gives the tier. - """ - - def needs_upgrade(self, tiers, flow_element): - - """! - Whether the tier of the data file the engine uses is not among a - property's tiers. - - @type tiers: list - @param tiers: the data tiers that hold the property - @type flow_element: FlowElement - @param flow_element: the engine - @rtype: bool - @return: True where an upgrade is needed + @raise Exception: always, saying why the property is missing """ - return flow_element.engine.getProduct() not in tiers + raise Exception("Property " + key + " not found in " + flow_element.datakey) diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_engines.py b/pipeline/fiftyone_pipeline_engines/tests/test_engines.py index 86553b9c..3b30e850 100644 --- a/pipeline/fiftyone_pipeline_engines/tests/test_engines.py +++ b/pipeline/fiftyone_pipeline_engines/tests/test_engines.py @@ -140,10 +140,7 @@ def testRestrictedProperty(self): except Exception as e: result = str(e) - self.assertEqual( - "Property 'integer' not found in data for element 'example'. " - "This is because the property has been excluded when configuring " - "the engine.", result) + self.assertTrue(result == "Property integer was excluded from example") # Restricted property names are compared ignoring case def testRestrictedPropertyIgnoresCase(self): @@ -167,9 +164,7 @@ def testRestrictedPropertyIgnoresCase(self): with self.assertRaises(Exception) as refused: flowdata.example.get(key) self.assertEqual( - "Property '" + key + "' not found in data for element " - "'example'. This is because the property has been excluded " - "when configuring the engine.", + "Property " + key + " was excluded from example", str(refused.exception)) # Test restricting properties @@ -188,10 +183,7 @@ def testMissingPropertyService(self): except Exception as e: result = str(e) - self.assertEqual( - "Property 'missing' not found in data for element 'example'. " - "The reason for this is unknown. Please check that the aspect and " - "property name are correct.", result) + self.assertTrue(result == "Property missing not found in example") # Test cache diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_missing_property_reasons.py b/pipeline/fiftyone_pipeline_engines/tests/test_missing_property_reasons.py deleted file mode 100644 index c9944ed4..00000000 --- a/pipeline/fiftyone_pipeline_engines/tests/test_missing_property_reasons.py +++ /dev/null @@ -1,121 +0,0 @@ -# ********************************************************************* -# This Original Work is copyright of 51 Degrees Mobile Experts Limited. -# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, -# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. -# -# This Original Work is licensed under the European Union Public Licence -# (EUPL) v.1.2 and is subject to its terms as set out below. -# -# If a copy of the EUPL was not distributed with this file, You can obtain -# one at https://opensource.org/licenses/EUPL-1.2. -# -# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be -# amended by the European Commission) shall be deemed incompatible for -# the purposes of the Work and the provisions of the compatibility -# clause in Article 5 of the EUPL shall not apply. -# -# If using the Work as, or as part of, a network application, by -# including the attribution notice(s) required under Article 5 of the EUPL -# in the end user terms of the application under an appropriate heading, -# such notice(s) shall fulfill the requirements of that article. -# ********************************************************************* - -""" -The reason an on-premise engine gives for a missing property, decided in -the order the properties specification sets out, with the message of the -missing properties table for that reason. -""" - -import unittest - -from fiftyone_pipeline_engines.missingproperty_service import \ - OnPremiseMissingPropertyService - -# The pricing page the data file rows link to, with this repository's -# campaign parameters. -PRICING = "https://51degrees.com/pricing?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-pricing" - - -class StandInNativeEngine: - """Stands in for the native engine, giving the data file's tier.""" - - def __init__(self, tier): - self.tier = tier - - def getProduct(self): - return self.tier - - -class StandInEngine: - """Stands in for an on-premise engine, with its property metadata keyed - by lower-case name as the engines keep it.""" - - def __init__(self, tier, properties): - self.datakey = "device" - self.engine = StandInNativeEngine(tier) - self.properties = properties - - def get_properties(self): - return self.properties - - -def message(name, engine): - with unittest.TestCase().assertRaises(Exception) as raised: - OnPremiseMissingPropertyService().check(name, engine) - return str(raised.exception) - - -class OnPremiseReasonTests(unittest.TestCase): - - def test_a_property_of_other_tiers_needs_an_upgrade(self): - engine = StandInEngine("Lite", { - "screenpixelsheight": {"datafiles": ["Premium", "Enterprise"]}}) - self.assertEqual( - "Property 'ScreenPixelsHeight' not found in data for element " - "'device'. This is because your license or data file does not " - "include this property. It is available with the " - "Premium,Enterprise license/data for the StandInEngine. To " - "upgrade, see " + PRICING + ".", - message("ScreenPixelsHeight", engine)) - - def test_a_property_with_no_tiers_known_needs_an_upgrade(self): - engine = StandInEngine("Lite", {"screenpixelsheight": { - "datafiles": []}}) - self.assertEqual( - "Property 'screenpixelsheight' not found in data for element " - "'device'. This is because your license or data file does not " - "include this property. To see which licenses include it, see " - + PRICING + ".", - message("screenpixelsheight", engine)) - - def test_a_property_of_this_tier_was_excluded(self): - engine = StandInEngine("Enterprise", { - "screenpixelsheight": {"datafiles": ["Premium", "Enterprise"]}}) - self.assertEqual( - "Property 'screenpixelsheight' not found in data for element " - "'device'. This is because the property has been excluded when " - "configuring the engine.", - message("screenpixelsheight", engine)) - - def test_a_property_the_metadata_does_not_list_is_unknown(self): - engine = StandInEngine("Enterprise", { - "screenpixelsheight": {"datafiles": ["Enterprise"]}}) - self.assertEqual( - "Property 'notthere' not found in data for element 'device'. " - "The reason for this is unknown. Please check that the aspect " - "and property name are correct.", - message("notthere", engine)) - - def test_names_are_compared_ignoring_case(self): - engine = StandInEngine("Lite", { - "screenpixelsheight": {"datafiles": ["Enterprise"]}}) - for name in ("screenpixelsheight", "SCREENPIXELSHEIGHT", - "ScreenPixelsHeight"): - with self.subTest(name=name): - self.assertIn( - "It is available with the Enterprise license/data", - message(name, engine)) - - -if __name__ == "__main__": - unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 88cb2f61..74ca6f9c 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -144,11 +144,10 @@ def __init__( needs cookie to be set, and works the same whether or not share_all is True. @type query_whitelist: list or str - @param query_whitelist: The query string and HTTP form parameters - to share, as well as those prefixed with '51D_', when - share_all_query_string_parameters is False. A string of names - separated by commas, which is how a configuration file gives a - list, is read as those names. None holds no names. + @param query_whitelist: Deprecated, and has no effect. A query + string or HTTP form parameter is shared only when its name starts + with '51D_' or the Pipeline uses it as evidence, whatever the + settings. @type header_blacklist: list or str @param header_blacklist: By default, all HTTP headers except Cookie are shared. @@ -160,13 +159,13 @@ def __init__( @type share_all: bool @param share_all: If True, all evidence is shared apart from the email address, the salt used for hashed email identifiers, the - request's query string and path, and every cookie whose name does - not start with '51D_'. False by default. + request's query string and path, every cookie whose name does not + start with '51D_', and every query string or HTTP form parameter + whose name does not start with '51D_' and that the Pipeline does + not use as evidence. False by default. @type share_all_query_string_parameters: bool - @param share_all_query_string_parameters: If True, every query - string and HTTP form parameter is shared. If False, only those - prefixed with '51D_' and those in query_whitelist are. True by - default. + @param share_all_query_string_parameters: Deprecated, and has no + effect, as query_whitelist. @type max_queue_size: int @param max_queue_size: The most entries held while waiting to be sent. When the queue is full, new entries are dropped. The default @@ -741,7 +740,7 @@ def process_internal(self, flow_data): # A hash of the values that would be shared, as the element # accepts no evidence of its own. cachekey = self.tracker_evidence_key_filter.filter_evidence( - self.found_evidence(flow_data)) + self.found_evidence(flow_data), flow_data.pipeline) # Evidence keys ignore case, so a repeat is found whatever the # letters of its keys. cachekey = json.dumps( @@ -784,7 +783,8 @@ def get_data_from_evidence(self, flow_data): f"The value '{value}' could not be parsed to an integer.") else: # Check if we can send this piece of evidence - if self.data_evidence_key_filter.filter_evidence_key(key): + if self.data_evidence_key_filter.filter_evidence_key( + key, flow_data.pipeline): data.try_add(key, value) return data diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py index 9438ff5a..e37fc724 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py @@ -21,6 +21,7 @@ # ********************************************************************* from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.javascriptbuilder import JavascriptBuilderElement # Cookies, session values and query string parameters whose names start with # this are the ones 51Degrees sets, and are shared. @@ -37,6 +38,30 @@ "id.email", "id.salt", "request-query", "request-path", "header.cookie") +# The query string parameters the JavaScript Builder's specification lists as +# its accepted Evidence. Its filter also passes every other parameter, only so +# that it can send them back to the callback. +JAVASCRIPT_BUILDER_PARAMETERS = ( + "query.fod-js-object-name", "query.fod-js-enable-cookies") + + +def _used_by(pipeline, key): + """! + Whether a FlowElement of the Pipeline accepts the key as evidence. The + JavaScript Builder is asked only about the parameters its specification + says it accepts. + """ + if pipeline is None: + return False + for element in pipeline.flow_elements: + if isinstance(element, JavascriptBuilderElement): + if key.lower() in JAVASCRIPT_BUILDER_PARAMETERS: + return True + elif element.get_evidence_key_filter().filter_evidence_key(key): + return True + return False + + def _split_names(names): """! A configuration file gives a list of names as one comma-separated @@ -53,10 +78,10 @@ class ShareUsageEvidenceKeyFilter(BasicListEvidenceKeyFilter): """ The ShareUsageEvidenceKeyFilter decides which evidence the 51Degrees ShareUsage service is sent. Cookies are shared only when their names - start with 51D_, whatever the other settings. Headers are shared unless - they are blocked, session values only when their names start with 51D_, - and query string and form parameters when all of them are shared or - when their names start with 51D_ or are on the list to share. Any other + start with 51D_, and query string and form parameters only when their + names start with 51D_ or the Pipeline uses them as evidence, whatever + the other settings. Headers are shared unless they are blocked, and + session values only when their names start with 51D_. Any other evidence is shared. """ @@ -73,8 +98,9 @@ def __init__( @type cookie: string @param cookie: the name of the cookie that identifies the session @type query_whitelist: list or str - @param query_whitelist: query string and form parameters to share as - well as those starting 51D_. None holds no names. A string is read as names separated by commas. + @param query_whitelist: deprecated, and has no effect. A query + string or form parameter is shared only when its name starts with + 51D_ or the Pipeline uses it as evidence. @type header_blacklist: list or str @param header_blacklist: headers not to share. A string is read as names separated by commas. @@ -84,14 +110,16 @@ def __init__( @param share_all: whether all evidence is shared, apart from the evidence that is never shared and cookies not starting 51D_ @type share_all_query_string_parameters: bool - @param share_all_query_string_parameters: whether every query - string and form parameter is shared, whatever query_whitelist holds + @param share_all_query_string_parameters: deprecated, and has no + effect, as query_whitelist. """ self.cookie = cookie.lower() if cookie else None query_whitelist = _split_names(query_whitelist) header_blacklist = _split_names(header_blacklist) + # Kept so that code reading them still works. Neither decides what + # is shared. self.query_whitelist = \ [parameter.lower() for parameter in query_whitelist or ()] @@ -105,18 +133,40 @@ def __init__( self.share_all_query_string_parameters = \ share_all_query_string_parameters - def filter_evidence_key(self, key): + def filter_evidence(self, evidence_key_object, pipeline=None): + + """! + The evidence to share, from a dictionary of evidence. + + @type evidence_key_object: dict + @param evidence_key_object: evidence keys and values + @type pipeline: Pipeline + @param pipeline: the Pipeline the evidence came to, whose evidence + decides which query string and form parameters are shared + @rtype: dict + @return: the evidence to share + """ + + return {key: value for key, value in evidence_key_object.items() + if self.filter_evidence_key(key, pipeline)} + + def filter_evidence_key(self, key, pipeline=None): """! Check if a specific key should be filtered. @type key: string @param key: to check in the filter + @type pipeline: Pipeline + @param pipeline: the Pipeline the evidence came to. Without one, a + query string or form parameter is shared only when its name starts + with 51D_. @rtype: bool @return: Is this key in the filter's keys list? """ + original = key key = key.lower() if any(key == never_shared or key.endswith("." + never_shared) @@ -133,6 +183,13 @@ def filter_evidence_key(self, key): return field.startswith(FIFTYONE_PREFIX) or \ (self.include_session and field == self.cookie) + # A query string or form parameter is shared only when 51Degrees set + # it or the Pipeline uses it as evidence, even when all evidence is, + # so that a value such as a password is never sent. + if separator and prefix == "query": + return field.startswith(FIFTYONE_PREFIX) or \ + _used_by(pipeline, original) + if self.share_all: return True @@ -146,12 +203,6 @@ def filter_evidence_key(self, key): # Only session values that start with the 51Degrees # cookie prefix return field.startswith(FIFTYONE_PREFIX) - elif prefix == "query": - # Share every parameter only when asked to. Otherwise only - # those starting 51d_ or in the list. - return self.share_all_query_string_parameters or \ - field.startswith(FIFTYONE_PREFIX) or \ - field in self.query_whitelist else: # Add anything that is not a cookie, header, session # variable or query parameter @@ -164,10 +215,10 @@ class ShareUsageTrackerEvidenceKeyFilter(ShareUsageEvidenceKeyFilter): out, because they make every request different. """ - def filter_evidence_key(self, key): + def filter_evidence_key(self, key, pipeline=None): if key.lower() in ("query.session-id", "query.sequence"): return False return super(ShareUsageTrackerEvidenceKeyFilter, self)\ - .filter_evidence_key(key) + .filter_evidence_key(key, pipeline) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py index 0256ca15..eb3a0fdc 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py @@ -22,10 +22,31 @@ import unittest +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.flowelement import FlowElement from fiftyone_pipeline_engines_fiftyone.share_usage_evidencekeyfilter import \ ShareUsageEvidenceKeyFilter +class UsingElement(FlowElement): + """Accepts the evidence keys given.""" + + def __init__(self, keys): + super(UsingElement, self).__init__() + self.keys = keys + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter(self.keys) + + +class UsingPipeline: + """Stands in for a Pipeline whose FlowElement uses the keys given.""" + + def __init__(self, *keys): + self.flow_elements = [UsingElement(list(keys))] + + class ShareUsageEvidenceKeyFilterTests(unittest.TestCase): def test_default_rules(self): @@ -53,20 +74,22 @@ def test_default_rules(self): def test_field_is_everything_after_the_first_dot(self): """! - Check that a name containing a dot is matched as a whole against - the parameters to share. + Check that a parameter whose name contains a dot is matched as a + whole against the evidence the Pipeline uses. """ - keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"]) + keys = ShareUsageEvidenceKeyFilter() + pipeline = UsingPipeline("query.valid.email") - self.assertTrue(keys.filter_evidence_key("query.valid.email")) - self.assertFalse(keys.filter_evidence_key("query.valid")) - self.assertFalse(keys.filter_evidence_key("query.other.name")) + self.assertTrue(keys.filter_evidence_key("query.valid.email", pipeline)) + self.assertFalse(keys.filter_evidence_key("query.valid", pipeline)) + self.assertFalse(keys.filter_evidence_key("query.other.name", pipeline)) def test_lists_ignore_case(self): """! - Check that the headers not to share and the parameters to share - are matched without regard to case. + Check that the headers not to share are matched without regard to + case, and that a parameter on the deprecated list to share is not + shared. """ keys = ShareUsageEvidenceKeyFilter( @@ -74,45 +97,55 @@ def test_lists_ignore_case(self): header_blacklist=["X-Secret"]) self.assertFalse(keys.filter_evidence_key("header.x-secret")) - self.assertTrue(keys.filter_evidence_key("query.campaign")) + self.assertFalse(keys.filter_evidence_key("query.campaign")) self.assertTrue(keys.filter_evidence_key("header.user-agent")) def test_lists_from_comma_separated_strings(self): """! Check that a string given for either list, the form a configuration file gives a list in, is read as names separated by - commas, each trimmed. + commas, each trimmed, and that the parameters named still are not + shared. """ keys = ShareUsageEvidenceKeyFilter( query_whitelist="Campaign, utm_source", header_blacklist="X-Secret,X-Other") - self.assertTrue(keys.filter_evidence_key("query.campaign")) - self.assertTrue(keys.filter_evidence_key("query.utm_source")) - self.assertFalse(keys.filter_evidence_key("query.c")) + self.assertEqual(["campaign", "utm_source"], keys.query_whitelist) + self.assertFalse(keys.filter_evidence_key("query.campaign")) + self.assertFalse(keys.filter_evidence_key("query.utm_source")) self.assertFalse(keys.filter_evidence_key("header.x-secret")) self.assertFalse(keys.filter_evidence_key("header.x-other")) self.assertTrue(keys.filter_evidence_key("header.x")) def test_no_parameter_list_shares_only_51d_parameters(self): """! - Check that None in place of the parameters to share holds no - names, as an empty list does, so only parameters starting 51D_ are - shared unless every parameter is. + Check that None in place of the deprecated parameters to share, and + an empty list, both share only parameters starting 51D_ when the + Pipeline uses no others. """ for query_whitelist in (None, []): - prefixed = ShareUsageEvidenceKeyFilter( - query_whitelist=query_whitelist) - every = ShareUsageEvidenceKeyFilter( - query_whitelist=query_whitelist, - share_all_query_string_parameters=True) + keys = ShareUsageEvidenceKeyFilter(query_whitelist=query_whitelist) - self.assertFalse(prefixed.filter_evidence_key("query.campaign")) - self.assertTrue( - prefixed.filter_evidence_key("query.51d_campaign")) - self.assertTrue(every.filter_evidence_key("query.campaign")) + self.assertFalse(keys.filter_evidence_key("query.campaign")) + self.assertTrue(keys.filter_evidence_key("query.51d_campaign")) + + def test_the_deprecated_option_changes_nothing(self): + """! + Check that the deprecated share_all_query_string_parameters option + shares nothing more, on or off, whatever the list. + """ + + for query_whitelist in (None, [], ["campaign"]): + for share_every in (False, True): + keys = ShareUsageEvidenceKeyFilter( + query_whitelist=query_whitelist, + share_all_query_string_parameters=share_every) + + self.assertFalse(keys.filter_evidence_key("query.campaign")) + self.assertTrue(keys.filter_evidence_key("query.51d_campaign")) def test_session_cookie(self): """! @@ -131,8 +164,9 @@ def test_session_cookie(self): def test_share_all(self): """! Check that sharing everything shares the evidence the other rules - leave out, apart from the evidence that is never shared and the - cookies that do not start 51D_. + leave out, apart from the evidence that is never shared, the + cookies that do not start 51D_, and the parameters that neither start + 51D_ nor are used by the Pipeline. """ keys = ShareUsageEvidenceKeyFilter(share_all=True) @@ -141,8 +175,9 @@ def test_share_all(self): self.assertFalse(keys.filter_evidence_key("cookie.sessionid")) self.assertTrue( keys.filter_evidence_key("cookie.51D_screenpixelswidth")) - self.assertTrue(keys.filter_evidence_key("query.password")) - self.assertTrue(keys.filter_evidence_key("query.id.usage")) + self.assertFalse(keys.filter_evidence_key("query.password")) + self.assertFalse(keys.filter_evidence_key("query.id.usage")) + self.assertTrue(keys.filter_evidence_key("query.51d_usage")) self.assertTrue(keys.filter_evidence_key("server.request-method")) self.assertFalse(keys.filter_evidence_key("query.id.email")) self.assertFalse(keys.filter_evidence_key("header.id.email")) @@ -153,18 +188,18 @@ def test_share_all(self): def test_every_query_parameter_when_sharing_them_all(self): """! - Check that sharing every query string parameter shares one not on - the list, and that a filter made without it keeps to the list. + Check that a parameter is shared when the Pipeline uses it, and + another is not, whatever the deprecated options say. """ every = ShareUsageEvidenceKeyFilter( - query_whitelist=["campaign"], - share_all_query_string_parameters=True) - listed = ShareUsageEvidenceKeyFilter(query_whitelist=["campaign"]) + query_whitelist=["password"], + share_all_query_string_parameters=True, + share_all=True) + pipeline = UsingPipeline("query.campaign") - self.assertTrue(every.filter_evidence_key("query.password")) - self.assertFalse(listed.filter_evidence_key("query.password")) - self.assertTrue(listed.filter_evidence_key("query.campaign")) + self.assertFalse(every.filter_evidence_key("query.password", pipeline)) + self.assertTrue(every.filter_evidence_key("query.campaign", pipeline)) def test_the_session_cookie_is_kept_for_the_tracker_alone(self): """! @@ -207,12 +242,13 @@ def test_never_shared_keys_match_whole_parts(self): never shared is treated as an ordinary key. """ - keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"]) + keys = ShareUsageEvidenceKeyFilter() - self.assertTrue(keys.filter_evidence_key("query.valid.email")) + self.assertTrue(keys.filter_evidence_key( + "query.valid.email", UsingPipeline("query.valid.email"))) self.assertTrue( ShareUsageEvidenceKeyFilter(share_all=True) - .filter_evidence_key("query.valid.email")) + .filter_evidence_key("header.valid.email")) def test_naming_a_parameter_does_not_share_the_query_string(self): """! @@ -220,7 +256,9 @@ def test_naming_a_parameter_does_not_share_the_query_string(self): string, which carries every parameter. """ - keys = ShareUsageEvidenceKeyFilter(query_whitelist=["utm_source"]) + keys = ShareUsageEvidenceKeyFilter() + pipeline = UsingPipeline("query.utm_source") - self.assertTrue(keys.filter_evidence_key("query.utm_source")) - self.assertFalse(keys.filter_evidence_key("server.request-query")) + self.assertTrue(keys.filter_evidence_key("query.utm_source", pipeline)) + self.assertFalse( + keys.filter_evidence_key("server.request-query", pipeline)) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py index 9209ae53..b31b9f1f 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py @@ -21,8 +21,9 @@ # ********************************************************************* """A configuration file gives a list of names as one comma-separated string, -so the usage sharing element takes its query string parameters to share and -its headers not to share in that form, as well as in a list.""" +so the usage sharing element takes its headers not to share in that form, as +well as in a list. Its deprecated query string parameters to share are read +the same way and shared no more than any other parameter.""" import json import unittest @@ -44,8 +45,7 @@ "endpoint": "http://127.0.0.1:9", } -# Shares only the query string parameters listed and those starting 51D_, -# where every one is shared by default. +# The deprecated option, which no longer changes what is shared. LISTED_ONLY = {"share_all_query_string_parameters": False} # A configuration file that gives both lists as comma-separated strings. @@ -114,8 +114,9 @@ def test_query_parameters_to_share_from_a_string(self): ShareUsage(query_whitelist="first,second", **LISTED_ONLY, **QUEUE_ONLY)) - self.assert_shared(shared, ("first value", "second value")) - self.assert_not_shared(shared, ("third value", "letter value")) + self.assert_not_shared( + shared, + ("first value", "second value", "third value", "letter value")) def test_headers_not_to_share_from_a_string(self): shared = shared_alone_by( @@ -130,10 +131,11 @@ def test_lists_from_a_configuration_file(self): usage = pipeline.get_element("shareusage") shared = shared_by(pipeline, usage) - self.assert_shared( - shared, ("first value", "second value", "agent value")) + self.assertEqual(["first", "second"], usage.query_whitelist) + self.assert_shared(shared, ("agent value",)) self.assert_not_shared( - shared, ("third value", "letter value", "one value", "two value")) + shared, ("first value", "second value", "third value", + "letter value", "one value", "two value")) def test_names_are_trimmed_and_empty_entries_left_out(self): usage = ShareUsage( @@ -165,10 +167,10 @@ def test_a_list_is_used_as_it_is(self): self.assertIs(names, usage.query_whitelist) self.assertIs(headers, usage.header_blacklist) shared = shared_alone_by(usage) - self.assert_shared( - shared, ("first value", "second value", "agent value")) + self.assert_shared(shared, ("agent value",)) self.assert_not_shared( - shared, ("third value", "letter value", "one value", "two value")) + shared, ("first value", "second value", "third value", + "letter value", "one value", "two value")) if __name__ == "__main__": diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py index 5df7e9be..40a4bd51 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py @@ -21,10 +21,11 @@ # ********************************************************************* """ -What usage sharing shares by default and when all Evidence is shared. -Every query string and form parameter is shared unless that is turned off, -no cookie but a 51D_ one is ever shared, and the session cookie is still -compared when sessions are tracked. +What usage sharing shares by default and when all Evidence is shared. A query +string or form parameter is shared only when its name starts with 51D_ or the +Pipeline uses it as evidence, whatever the settings, no cookie but a 51D_ one +is ever shared, and the session cookie is still compared when sessions are +tracked. """ import gzip @@ -36,6 +37,8 @@ import requests from flask import Flask, request +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter from fiftyone_pipeline_core.flowelement import FlowElement from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_pipeline_core.web import webevidence @@ -53,8 +56,8 @@ "endpoint": "http://127.0.0.1:9", } -# Turns off sharing every query string parameter, from a configuration -# file. +# The deprecated option to share every query string parameter, from a +# configuration file. CONFIGURATION = """ { "PipelineOptions": { @@ -104,50 +107,91 @@ def records_of(usage, requests_evidence, *elements): return sent_on_close(usage) -class QueryAndFormValuesTests(unittest.TestCase): +class UsingElement(FlowElement): + """Accepts the query string parameter given as evidence, and adds no + data.""" + + def __init__(self, key): + super(UsingElement, self).__init__() + self.datakey = "using" + self.key = key + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter([self.key]) + + +# Every way the deprecated options could once share more. +DEPRECATED_SETTINGS = ( + {}, + {"share_all": True}, + {"query_whitelist": ["password", "campaign"]}, + {"share_all_query_string_parameters": True}, + {"share_all_query_string_parameters": False, + "query_whitelist": "password,campaign"}, +) + + +def shared_from_web_request(settings, method): + """The record usage sharing sends for one web request with the + parameters below, in its query string for GET or its form body for + POST, through a Pipeline whose other element uses query.used.""" + app = Flask(__name__) + usage = ShareUsage(**dict(QUEUE_ONLY, **settings)) + pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS) \ + .add(UsingElement("query.used")).add(usage).build() + parameters = {"51D_screen": "fiftyone value", "used": "used value", + "password": "password value", "campaign": "campaign value"} + if method == "GET": + context = app.test_request_context("/", query_string=parameters) + else: + context = app.test_request_context( + "/", method="POST", data=parameters) + with context: + flowdata = pipeline.create_flowdata() + flowdata.evidence.add_from_dict(webevidence(request)) + flowdata.process() + records = sent_on_close(usage) + return records[0] if records else "" - def test_every_query_value_is_shared_by_default(self): - records = records_of(ShareUsage(**QUEUE_ONLY), [{ - "header.user-agent": "agent value", - "query.campaign": "campaign value", - "query.password": "password value", - }]) - self.assertEqual(1, len(records)) - self.assertIn('campaign value', - records[0]) - self.assertIn('password value', - records[0]) +class QueryAndFormValuesTests(unittest.TestCase): - def test_every_form_value_of_the_web_request_is_shared_by_default(self): - app = Flask(__name__) + def test_a_51d_parameter_is_shared_from_a_query_string_and_a_form(self): + for method in ("GET", "POST"): + with self.subTest(method=method): + record = shared_from_web_request({}, method) + self.assertIn("fiftyone value", record) + + def test_a_parameter_the_pipeline_uses_is_shared_from_both(self): + for method in ("GET", "POST"): + with self.subTest(method=method): + record = shared_from_web_request({}, method) + self.assertIn('used value', record) + + def test_no_other_parameter_is_ever_shared_from_either(self): + for settings in DEPRECATED_SETTINGS: + for method in ("GET", "POST"): + with self.subTest(settings=settings, method=method): + record = shared_from_web_request(settings, method) + self.assertIn("fiftyone value", record) + self.assertNotIn("password value", record) + self.assertNotIn("campaign value", record) + + def test_the_javascript_builder_uses_only_the_parameters_it_lists(self): usage = ShareUsage(**QUEUE_ONLY) - pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS).add(usage).build() - with app.test_request_context( - "/", method="POST", data={"field": "form value"}): - flowdata = pipeline.create_flowdata() - flowdata.evidence.add_from_dict(webevidence(request)) - flowdata.process() + pipeline = PipelineBuilder().add(usage).build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "agent value") + flowdata.evidence.add("query.password", "password value") + flowdata.evidence.add("query.fod-js-object-name", "object value") + flowdata.process() records = sent_on_close(usage) - self.assertEqual(1, len(records)) - self.assertIn('form value', records[0]) - - def test_turned_off_only_51d_and_listed_values_are_shared(self): - records = records_of(ShareUsage( - share_all_query_string_parameters=False, - query_whitelist=["campaign"], **QUEUE_ONLY), [{ - "header.user-agent": "agent value", - "query.campaign": "campaign value", - "query.51d_gethighentropyvalues": "hev", - "query.password": "password value", - }]) - - self.assertIn("campaign value", records[0]) - self.assertIn("hev", records[0]) + self.assertIn("agent value", records[0]) + self.assertIn("object value", records[0]) self.assertNotIn("password value", records[0]) - def test_turned_off_from_a_configuration_file(self): + def test_the_deprecated_option_from_a_configuration_file(self): pipeline = PipelineBuilder().build_from_configuration( json.loads(CONFIGURATION)) usage = pipeline.get_element("shareusage") @@ -161,7 +205,6 @@ def test_turned_off_from_a_configuration_file(self): self.assertIn("agent value", records[0]) self.assertNotIn("password value", records[0]) - class CookieTests(unittest.TestCase): def test_only_51d_cookies_are_shared_when_all_evidence_is_shared(self): diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index c415387e..c3749389 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -38,6 +38,22 @@ import requests from unittest import mock +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.flowelement import FlowElement + + +class UsingElement(FlowElement): + """Accepts the evidence keys given, and adds no data.""" + + def __init__(self, keys): + super(UsingElement, self).__init__() + self.datakey = "using" + self.keys = keys + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter(self.keys) + testHost = "127.0.0.1" testPort = 5000 testEndpoint = f"http://{testHost}:{testPort}" @@ -919,9 +935,9 @@ def test_escapes_xml_characters(self): def test_default_shares_only_accepted_evidence(self): """! - Check that with no options the element shares headers, the - 51Degrees cookies and every query string and form parameter, and - leaves out other cookies, the Cookie header and the email address. + Check that with no options the element shares headers and the + 51Degrees cookies and parameters, and leaves out other cookies, the + Cookie header, and other query string and form parameters. """ usage = ShareUsage( @@ -948,7 +964,7 @@ def test_default_shares_only_accepted_evidence(self): self.assertIn('hev', app.received[0]) self.assertNotIn("cookie value", app.received[0]) self.assertNotIn("header value", app.received[0]) - self.assertIn('password value', app.received[0]) + self.assertNotIn("password value", app.received[0]) self.assertNotIn("email value", app.received[0]) def test_never_shared_evidence(self): @@ -984,8 +1000,9 @@ def test_never_shared_evidence(self): def test_share_all(self): """! Check that sharing all evidence shares what the other rules leave - out, but not the evidence that is never shared or a cookie that - does not start 51D_. + out, but not the evidence that is never shared, a cookie that + does not start 51D_, or a parameter that neither starts 51D_ nor is + used by the Pipeline. """ usage = ShareUsage( @@ -1007,22 +1024,21 @@ def test_share_all(self): self.assertEqual(len(app.received), 1) self.assertNotIn("cookie value", app.received[0]) self.assertIn('1080', app.received[0]) - self.assertIn('campaign value', app.received[0]) + self.assertNotIn("campaign value", app.received[0]) self.assertNotIn("email value", app.received[0]) def test_parameter_names_with_dots(self): """! Check that a query string parameter whose name contains a dot is - matched by its whole name against the parameters to share. + matched by its whole name against the evidence the Pipeline uses. """ usage = ShareUsage( requested_package_size=1, - endpoint=testEndpoint, - share_all_query_string_parameters=False, - query_whitelist=["valid.email"]) + endpoint=testEndpoint) pipeline = (PipelineBuilder())\ + .add(UsingElement(["query.valid.email"]))\ .add(usage)\ .build() flowdata = pipeline.create_flowdata()