From f03482ca65adc84c8ad36c700e51b05de60b47c1 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 21:22:26 +0000 Subject: [PATCH 1/7] Test the released missing property messages and a parameter list of None Restore the released form of the tests whose behaviour rests on a specification SHOULD, being the missing property messages and a usage sharing filter given None for its query string parameters: - test_engines.py testRestrictedProperty, testRestrictedPropertyIgnoresCase and testMissingPropertyService - the on-premise Device Detection test_devicedetection.py test_excluded_property, test_missing_property_service_not_found_anywhere and test_missing_property_service_not_found_in_current - test_credentials.py test_missing_property_message_names_both_causes - the cloud Device Detection test_cloud.py test_missing_property_service_not_found_anywhere - test_share_usage_evidencekeyfilter.py test_no_parameter_list_shares_every_parameter test_missing_property_reasons.py and test_cloudengine_missing_property.py go, because each of their checks is a message of the missing properties table, which the released messages replace. New tests keep what a MUST requires. In both on-premise packages a property the engine lists, in a data file tier that holds it, raises when the data has no value for it. And share_all_query_string_parameters given as False or True decides whatever the parameter list holds. --- .../tests/test_cloud.py | 16 +-- .../tests/test_devicedetection.py | 31 ++-- .../tests/test_missing_property_raises.py | 56 ++++++++ .../tests/test_missing_property_raises.py | 56 ++++++++ .../test_cloudengine_missing_property.py | 133 ------------------ .../tests/test_credentials.py | 20 +-- .../tests/test_engines.py | 14 +- .../tests/test_missing_property_reasons.py | 121 ---------------- .../test_share_usage_evidencekeyfilter.py | 25 +++- 9 files changed, 160 insertions(+), 312 deletions(-) create mode 100644 device-detection/fiftyone_devicedetection_onpremise/tests/test_missing_property_raises.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_missing_property_raises.py delete mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_property.py delete mode 100644 pipeline/fiftyone_pipeline_engines/tests/test_missing_property_reasons.py diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py index 702e5c04..7d769b84 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py @@ -30,8 +30,6 @@ from fiftyone_devicedetection_cloud.devicedetection_cloud import DeviceDetectionCloud -from fiftyone_pipeline_engines.missingproperty_service import CONFIGURATOR_URL - from .live_cloud import find_resource_key, no_resource_key pytestmark = pytest.mark.live @@ -158,16 +156,10 @@ def test_missing_property_service_not_found_anywhere(self): fd.device.get("notpresent") result = str(raised.exception) - properties = pipeline.get_element("device").get_properties() - self.assertIn( - "Property 'notpresent' not found in data for element 'device'. " - "This is because your resource key does not include access to " - "this property. The properties included for your key under " - "'device' are ", result) - self.assertIn( - ",".join(meta.get("name", name) - for name, meta in properties.items()), result) - self.assertIn(CONFIGURATOR_URL, result) + self.assertIn("notpresent", result) + self.assertIn("device", result) + self.assertIn(', '.join(pipeline.get_element("device").get_properties().keys()), result) + self.assertIn("https://51degrees.com/documentation/_services__cloud__resource_keys.html", result) def test_cloud_request_origin(self): """! diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py index aa6222ca..f8151eaf 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py @@ -261,9 +261,7 @@ def test_missing_property_service_not_found_anywhere(self): result = str(e) self.assertEqual( - "Property 'notpresent' not found in data for element 'device'. " - "The reason for this is unknown. Please check that the aspect and " - "property name are correct.", result) + result, "Property notpresent not found in data for element device. Please check that the element and property names are correct.") def test_missing_property_service_not_found_in_current(self): """! @@ -288,20 +286,15 @@ def test_missing_property_service_not_found_in_current(self): fd.process() - # The name is found however it is spelled, and the message gives it - # as the caller did. - for name in ("mock", "Mock"): - with self.assertRaises(Exception) as raised: - fd.device.get(name) - self.assertEqual( - "Property '" + name + "' not found in data for element " - "'device'. This is because your license or data file does not " - "include this property. It is available with the Enterprise " - "license/data for the DeviceDetectionOnPremise. To upgrade, " - "see " - "https://51degrees.com/pricing?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-pricing" - ".", - str(raised.exception)) + result = "" + + try: + fd.device.get("mock") + except Exception as e: + result = str(e) + + self.assertEqual( + result, "Property mock not found in data for element device. This is because your datafile does not contain the property. The property is available in['Enterprise']") def test_excluded_property(self): """! @@ -329,9 +322,7 @@ def test_excluded_property(self): result = str(e) self.assertEqual( - "Property 'screenpixelsheight' not found in data for element " - "'device'. This is because the property has been excluded when " - "configuring the engine.", result) + result, "Property screenpixelsheight was excluded from device") def test_evidencekey_filter_contains_user_agent(self): """! diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_missing_property_raises.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_missing_property_raises.py new file mode 100644 index 00000000..2fe00eaa --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_missing_property_raises.py @@ -0,0 +1,56 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""A property the engine lists, in a data file tier that holds it, raises +when the data has no value for it, as every missing property does.""" + +import unittest +from types import SimpleNamespace + +from fiftyone_devicedetection_onpremise.swig_data import \ + OnPremiseMissingPropertyService + + +class StandInEngine: + """An engine whose data file is Lite, listing one property that the + Lite file holds.""" + + datakey = "device" + properties = {"ismobile": {"datafiles": ["Lite", "Enterprise"]}} + engine = SimpleNamespace(getProduct=lambda: "Lite") + + +class MissingPropertyRaisesTests(unittest.TestCase): + + def test_a_listed_property_of_this_tier_raises(self): + with self.assertRaises(Exception) as raised: + OnPremiseMissingPropertyService().check( + "ismobile", StandInEngine()) + + self.assertEqual( + "Property 'ismobile' not found in data for element 'device'. " + "This is because the property has been excluded when " + "configuring the engine.", str(raised.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_missing_property_raises.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_missing_property_raises.py new file mode 100644 index 00000000..f8e282e7 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_missing_property_raises.py @@ -0,0 +1,56 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""A property the engine lists, in a data file tier that holds it, raises +when the data has no value for it, as every missing property does.""" + +import unittest +from types import SimpleNamespace + +from fiftyone_ipintelligence_onpremise.swig_data import \ + OnPremiseMissingPropertyService + + +class StandInEngine: + """An engine whose data file is Lite, listing one property that the + Lite file holds.""" + + datakey = "ip" + properties = {"registeredname": {"datafiles": ["Lite", "Enterprise"]}} + engine = SimpleNamespace(getProduct=lambda: "Lite") + + +class MissingPropertyRaisesTests(unittest.TestCase): + + def test_a_listed_property_of_this_tier_raises(self): + with self.assertRaises(Exception) as raised: + OnPremiseMissingPropertyService().check( + "registeredname", StandInEngine()) + + self.assertEqual( + "Property 'registeredname' not found in data for element 'ip'. " + "This is because the property has been excluded when " + "configuring the engine.", str(raised.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_property.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_property.py deleted file mode 100644 index 92117abc..00000000 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_property.py +++ /dev/null @@ -1,133 +0,0 @@ -# ********************************************************************* -# This Original Work is copyright of 51 Degrees Mobile Experts Limited. -# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, -# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. -# -# This Original Work is licensed under the European Union Public Licence -# (EUPL) v.1.2 and is subject to its terms as set out below. -# -# If a copy of the EUPL was not distributed with this file, You can obtain -# one at https://opensource.org/licenses/EUPL-1.2. -# -# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be -# amended by the European Commission) shall be deemed incompatible for -# the purposes of the Work and the provisions of the compatibility -# clause in Article 5 of the EUPL shall not apply. -# -# If using the Work as, or as part of, a network application, by -# including the attribution notice(s) required under Article 5 of the EUPL -# in the end user terms of the application under an appropriate heading, -# such notice(s) shall fulfill the requirements of that article. -# ********************************************************************* - -""" -The reason a Cloud Aspect Engine gives for a missing property, decided in -the order the properties specification sets out, with the message of the -missing properties table for that reason. -""" - -import json -import unittest - -from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine -from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary -from fiftyone_pipeline_core.flowelement import FlowElement -from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder - -# The configurator the Resource Key rows link to, with this repository's -# campaign parameters. -CONFIGURATOR = "https://configure.51degrees.com/?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-configurator" - - -class FakeCloudRequestEngine(FlowElement): - """Stands in for the Cloud Request Engine of a key that carries only the - device product, with two of its properties.""" - - def __init__(self): - super(FakeCloudRequestEngine, self).__init__() - self.datakey = "cloud" - self.flow_element_properties = {"device": { - "ismobile": {"name": "IsMobile", "type": "Boolean"}, - "hardwarevendor": {"name": "HardwareVendor", "type": "String"}}} - - def process_internal(self, flowdata): - flowdata.set_element_data(ElementDataDictionary( - self, {"cloud": json.dumps({"device": {"ismobile": True}})})) - - -class UndiscoveredCloudRequestEngine(FakeCloudRequestEngine): - """Stands in for a Cloud Request Engine whose request for the property - list failed when it was built, so its properties are not known.""" - - def known_flow_element_properties(self): - return None - - -def build(request_engine, datakey, restricted=None): - engine = CloudEngine() - engine.datakey = datakey - if restricted is not None: - engine.set_restricted_properties(restricted) - pipeline = PipelineBuilder({ - "add_javascript_builder": False, - "use_setheader_properties": False}).add( - request_engine).add(engine).build() - flowdata = pipeline.create_flowdata() - flowdata.process() - return pipeline, flowdata - - -class CloudReasonTests(unittest.TestCase): - - def message(self, request_engine, datakey, name, restricted=None): - pipeline, flowdata = build(request_engine, datakey, restricted) - self.addCleanup(pipeline.close) - with self.assertRaises(Exception) as raised: - flowdata.get(datakey).get(name) - return str(raised.exception) - - def test_a_product_the_key_does_not_include(self): - self.assertEqual( - "Property 'country' not found in data for element 'location'. " - "This is because your resource key does not include access to " - "any properties under 'location'. Check or create a resource key " - "with the properties you need in the configurator at " - + CONFIGURATOR + ".", - self.message(FakeCloudRequestEngine(), "location", "country")) - - def test_a_property_the_key_does_not_include(self): - self.assertEqual( - "Property 'browsername' not found in data for element 'device'. " - "This is because your resource key does not include access to " - "this property. The properties included for your key under " - "'device' are IsMobile,HardwareVendor. Check or change your key " - "in the configurator at " + CONFIGURATOR + ".", - self.message(FakeCloudRequestEngine(), "device", "browsername")) - - def test_a_listed_property_not_in_the_answer_was_excluded(self): - self.assertEqual( - "Property 'HardwareVendor' not found in data for element " - "'device'. This is because the property has been excluded when " - "configuring the engine.", - self.message( - FakeCloudRequestEngine(), "device", "HardwareVendor")) - - def test_a_property_left_out_of_the_restricted_list_was_excluded(self): - self.assertIn( - "This is because the property has been excluded when " - "configuring the engine.", - self.message( - FakeCloudRequestEngine(), "device", "hardwarevendor", - restricted=["IsMobile"])) - - def test_a_failed_request_for_the_property_list(self): - self.assertEqual( - "Property 'browsername' not found in data for element 'device'. " - "This is because an upstream cloud request failed, so no data " - "was available to populate the property.", - self.message( - UndiscoveredCloudRequestEngine(), "device", "browsername")) - - -if __name__ == "__main__": - unittest.main() diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py index 1f42abc6..72b5a11c 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py @@ -595,12 +595,13 @@ def test_no_report_when_every_property_came_back(self): str(warning.message) for warning in caught if "not covered" in str(warning.message)]) - def test_missing_property_message_names_the_properties_carried(self): + def test_missing_property_message_names_both_causes(self): """ - With a licence key, a property the key does not carry gives the - missing properties table's message for it, which lists the - properties the key carries under the element it looked in rather - than assuming device. + With a licence key the accessible properties list everything the + licence entitles, so a property can be listed as carried and still + be absent from an answer because it was not requested. The message + for a property that is not in the data says so, and names the + element it looked in rather than assuming device. """ engine = CloudRequestEngine({ @@ -620,8 +621,7 @@ def test_missing_property_message_names_the_properties_carried(self): data.device.get("other") message = str(context.exception) - self.assertIn( - "Property 'other' not found in data for element 'device'. This " - "is because your resource key does not include access to this " - "property. The properties included for your key under 'device' " - "are value.", message) + self.assertIn("Property other not found", message) + self.assertIn("requested_properties", message) + self.assertIn("under device are value", message) + self.assertNotIn("your resource key", message) diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_engines.py b/pipeline/fiftyone_pipeline_engines/tests/test_engines.py index 86553b9c..3b30e850 100644 --- a/pipeline/fiftyone_pipeline_engines/tests/test_engines.py +++ b/pipeline/fiftyone_pipeline_engines/tests/test_engines.py @@ -140,10 +140,7 @@ def testRestrictedProperty(self): except Exception as e: result = str(e) - self.assertEqual( - "Property 'integer' not found in data for element 'example'. " - "This is because the property has been excluded when configuring " - "the engine.", result) + self.assertTrue(result == "Property integer was excluded from example") # Restricted property names are compared ignoring case def testRestrictedPropertyIgnoresCase(self): @@ -167,9 +164,7 @@ def testRestrictedPropertyIgnoresCase(self): with self.assertRaises(Exception) as refused: flowdata.example.get(key) self.assertEqual( - "Property '" + key + "' not found in data for element " - "'example'. This is because the property has been excluded " - "when configuring the engine.", + "Property " + key + " was excluded from example", str(refused.exception)) # Test restricting properties @@ -188,10 +183,7 @@ def testMissingPropertyService(self): except Exception as e: result = str(e) - self.assertEqual( - "Property 'missing' not found in data for element 'example'. " - "The reason for this is unknown. Please check that the aspect and " - "property name are correct.", result) + self.assertTrue(result == "Property missing not found in example") # Test cache diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_missing_property_reasons.py b/pipeline/fiftyone_pipeline_engines/tests/test_missing_property_reasons.py deleted file mode 100644 index c9944ed4..00000000 --- a/pipeline/fiftyone_pipeline_engines/tests/test_missing_property_reasons.py +++ /dev/null @@ -1,121 +0,0 @@ -# ********************************************************************* -# This Original Work is copyright of 51 Degrees Mobile Experts Limited. -# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, -# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. -# -# This Original Work is licensed under the European Union Public Licence -# (EUPL) v.1.2 and is subject to its terms as set out below. -# -# If a copy of the EUPL was not distributed with this file, You can obtain -# one at https://opensource.org/licenses/EUPL-1.2. -# -# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be -# amended by the European Commission) shall be deemed incompatible for -# the purposes of the Work and the provisions of the compatibility -# clause in Article 5 of the EUPL shall not apply. -# -# If using the Work as, or as part of, a network application, by -# including the attribution notice(s) required under Article 5 of the EUPL -# in the end user terms of the application under an appropriate heading, -# such notice(s) shall fulfill the requirements of that article. -# ********************************************************************* - -""" -The reason an on-premise engine gives for a missing property, decided in -the order the properties specification sets out, with the message of the -missing properties table for that reason. -""" - -import unittest - -from fiftyone_pipeline_engines.missingproperty_service import \ - OnPremiseMissingPropertyService - -# The pricing page the data file rows link to, with this repository's -# campaign parameters. -PRICING = "https://51degrees.com/pricing?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-pricing" - - -class StandInNativeEngine: - """Stands in for the native engine, giving the data file's tier.""" - - def __init__(self, tier): - self.tier = tier - - def getProduct(self): - return self.tier - - -class StandInEngine: - """Stands in for an on-premise engine, with its property metadata keyed - by lower-case name as the engines keep it.""" - - def __init__(self, tier, properties): - self.datakey = "device" - self.engine = StandInNativeEngine(tier) - self.properties = properties - - def get_properties(self): - return self.properties - - -def message(name, engine): - with unittest.TestCase().assertRaises(Exception) as raised: - OnPremiseMissingPropertyService().check(name, engine) - return str(raised.exception) - - -class OnPremiseReasonTests(unittest.TestCase): - - def test_a_property_of_other_tiers_needs_an_upgrade(self): - engine = StandInEngine("Lite", { - "screenpixelsheight": {"datafiles": ["Premium", "Enterprise"]}}) - self.assertEqual( - "Property 'ScreenPixelsHeight' not found in data for element " - "'device'. This is because your license or data file does not " - "include this property. It is available with the " - "Premium,Enterprise license/data for the StandInEngine. To " - "upgrade, see " + PRICING + ".", - message("ScreenPixelsHeight", engine)) - - def test_a_property_with_no_tiers_known_needs_an_upgrade(self): - engine = StandInEngine("Lite", {"screenpixelsheight": { - "datafiles": []}}) - self.assertEqual( - "Property 'screenpixelsheight' not found in data for element " - "'device'. This is because your license or data file does not " - "include this property. To see which licenses include it, see " - + PRICING + ".", - message("screenpixelsheight", engine)) - - def test_a_property_of_this_tier_was_excluded(self): - engine = StandInEngine("Enterprise", { - "screenpixelsheight": {"datafiles": ["Premium", "Enterprise"]}}) - self.assertEqual( - "Property 'screenpixelsheight' not found in data for element " - "'device'. This is because the property has been excluded when " - "configuring the engine.", - message("screenpixelsheight", engine)) - - def test_a_property_the_metadata_does_not_list_is_unknown(self): - engine = StandInEngine("Enterprise", { - "screenpixelsheight": {"datafiles": ["Enterprise"]}}) - self.assertEqual( - "Property 'notthere' not found in data for element 'device'. " - "The reason for this is unknown. Please check that the aspect " - "and property name are correct.", - message("notthere", engine)) - - def test_names_are_compared_ignoring_case(self): - engine = StandInEngine("Lite", { - "screenpixelsheight": {"datafiles": ["Enterprise"]}}) - for name in ("screenpixelsheight", "SCREENPIXELSHEIGHT", - "ScreenPixelsHeight"): - with self.subTest(name=name): - self.assertIn( - "It is available with the Enterprise license/data", - message(name, engine)) - - -if __name__ == "__main__": - unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py index 0256ca15..f94a4d90 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py @@ -95,16 +95,31 @@ def test_lists_from_comma_separated_strings(self): self.assertFalse(keys.filter_evidence_key("header.x-other")) self.assertTrue(keys.filter_evidence_key("header.x")) - def test_no_parameter_list_shares_only_51d_parameters(self): + def test_no_parameter_list_shares_every_parameter(self): """! - Check that None in place of the parameters to share holds no - names, as an empty list does, so only parameters starting 51D_ are - shared unless every parameter is. + Check that None in place of the parameters to share shares every + parameter, and an empty list only those starting 51D_. + """ + + every = ShareUsageEvidenceKeyFilter(query_whitelist=None) + prefixed = ShareUsageEvidenceKeyFilter(query_whitelist=[]) + + self.assertTrue(every.filter_evidence_key("query.campaign")) + self.assertFalse(prefixed.filter_evidence_key("query.campaign")) + self.assertTrue(prefixed.filter_evidence_key("query.51d_campaign")) + + def test_the_option_given_decides_whatever_the_list(self): + """! + Check that with share_all_query_string_parameters given, None in + place of the parameters to share holds no names, as an empty list + does, so the option off shares only parameters starting 51D_ and + the option on shares every parameter. """ for query_whitelist in (None, []): prefixed = ShareUsageEvidenceKeyFilter( - query_whitelist=query_whitelist) + query_whitelist=query_whitelist, + share_all_query_string_parameters=False) every = ShareUsageEvidenceKeyFilter( query_whitelist=query_whitelist, share_all_query_string_parameters=True) From 9c1b7fd049518d40b991b26439daa8784c29ae0f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 21:24:49 +0000 Subject: [PATCH 2/7] Give the released missing property messages again The properties specification asks that an implementation SHOULD give the missing properties table's text and SHOULD decide the reason in the table's order. As both are SHOULDs, each engine raises the message it raised in the last release again, so the released tests pass unaltered. That restores the missing property services of the engines, cloud and on-premise packages and AspectData.get to their released form. The one change that stays is the specification's MUST to raise for a missing property. The on-premise Device Detection and IP Intelligence services raise for a property the engine lists in a data file tier that holds it, with the table's message for a property that is not available, where they returned nothing. --- .../hardwareprofile_cloud.py | 4 +- .../swig_data.py | 19 +- .../swig_data.py | 21 +- .../clouddata.py | 65 ++---- .../cloudengine.py | 26 +-- .../fiftyone_pipeline_engines/aspectdata.py | 11 +- .../missingproperty_service.py | 215 +----------------- 7 files changed, 78 insertions(+), 283 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py index a19b7da2..b6e6fe5e 100644 --- a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py @@ -23,7 +23,7 @@ from __future__ import absolute_import from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine from fiftyone_pipeline_cloudrequestengine.clouddata import \ - CloudMissingPropertyService + OnPremiseMissingPropertyService from fiftyone_pipeline_engines.aspectdata import AspectData from fiftyone_pipeline_engines.aspectdata_dictionary import AspectDataDictionary @@ -91,7 +91,7 @@ def __init__(self, flow_element, contents): # The device data's service, so a name that is not a property is # reported in the same way for both. - self.missing_property_service = CloudMissingPropertyService() + self.missing_property_service = OnPremiseMissingPropertyService() def _matching_key(self, key): """! diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index 05f4cc0f..7d7891fa 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -25,8 +25,7 @@ from fiftyone_pipeline_engines.aspectdata import AspectData from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue -from fiftyone_pipeline_engines.missingproperty_service import \ - OnPremiseMissingPropertyService +from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService from .constants import RESULTS_CLOSED @@ -34,6 +33,22 @@ # fiftyoneDegreesHashMatchMethod values, which getMethod returns. MATCH_METHODS = ("NONE", "PERFORMANCE", "COMBINED", "PREDICTIVE") +class OnPremiseMissingPropertyService(MissingPropertyService): + + def check(self, key, flow_element): + + if key in flow_element.properties: + datafiles = flow_element.properties[key]["datafiles"] + if(not flow_element.engine.getProduct() in datafiles): + raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. This is because your datafile does not contain the property. The property is available in' + str(datafiles)) + # The data file holds the property, so it is missing because + # it is not available. A missing property always raises. + raise Exception( + "Property '" + key + "' not found in data for element '" + + flow_element.datakey + "'. This is because the property " + "has been excluded when configuring the engine.") + else: + raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. Please check that the element and property names are correct.') class SwigData(AspectData): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index 2070a22b..85fa7094 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -27,8 +27,7 @@ from fiftyone_pipeline_engines.aspectdata import AspectData from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue -from fiftyone_pipeline_engines.missingproperty_service import \ - OnPremiseMissingPropertyService +from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService from .constants import RESULTS_CLOSED @@ -43,6 +42,24 @@ COORDINATE_PROPERTIES = ("latitude", "longitude") +class OnPremiseMissingPropertyService(MissingPropertyService): + + def check(self, key, flow_element): + + if key in flow_element.properties: + datafiles = flow_element.properties[key]["datafiles"] + if(not flow_element.engine.getProduct() in datafiles): + raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. This is because your datafile does not contain the property. The property is available in' + str(datafiles)) + # The data file holds the property, so it is missing because + # it is not available. A missing property always raises. + raise Exception( + "Property '" + key + "' not found in data for element '" + + flow_element.datakey + "'. This is because the property " + "has been excluded when configuring the engine.") + else: + raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. Please check that the element and property names are correct.') + + class SwigData(AspectData): """! diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py index e45f67fd..4f4f8c88 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py @@ -22,48 +22,27 @@ from fiftyone_pipeline_engines.aspectdata_dictionary import AspectDataDictionary -from fiftyone_pipeline_engines.missingproperty_service import \ - MissingPropertyReason, MissingPropertyService - - -class CloudMissingPropertyService(MissingPropertyService): - - """! - The missing property service of a Cloud Aspect Engine. A property its - metadata does not list is one the key in use does not include, or, - while the engine's property list is not known, one whose request for - that list failed. - """ - - def engine_reason(self, flow_element): - - """! - The reason for a property the metadata does not list. - - @type flow_element: CloudEngine - @param flow_element: the engine - @rtype: tuple - @return: the reason, and the property names the key includes - """ - - properties_known = getattr(flow_element, "properties_known", None) - if properties_known is not None and not properties_known(): - # The Cloud Request Engine asks for the property list when it is - # built, so a list that is still not known is one whose request - # failed. - return MissingPropertyReason.CLOUD_REQUEST_FAILED, () - properties = self.properties(flow_element) - if not properties: - return ( - MissingPropertyReason.PRODUCT_NOT_ACCESSIBLE_WITH_RESOURCE_KEY, - ()) - return ( - MissingPropertyReason.PROPERTY_NOT_ACCESSIBLE_WITH_RESOURCE_KEY, - [meta.get("name", name) for name, meta in properties.items()]) - - -# The name this service had before, kept for code that imports it. -OnPremiseMissingPropertyService = CloudMissingPropertyService +from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService + +class OnPremiseMissingPropertyService(MissingPropertyService): + + def check(self, key, flow_element): + + # The properties listed here are the ones the key carries, which is + # not the same as the ones one answer holds: a request made with a + # licence key carries only the properties it asked for, so a + # property can be listed and still be absent. The message names + # both causes, so the reader checks the right one. + raise Exception( + "Property " + key + " not found in data for element " + + flow_element.datakey + ". Either the key in use does not " + "include access to this property, or the request was made " + "with a licence key and the property was not among the " + "requested_properties. Properties the key carries under " + + flow_element.datakey + " are " + + ", ".join(list(flow_element.get_properties().keys())) + + ". For more details on resource keys, see our explainer: " + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-clouddata.py&utm_term=property-not-found") class CloudData(AspectDataDictionary): @@ -83,4 +62,4 @@ def __init__(self, flow_element, contents): super(CloudData, self).__init__(flow_element, contents) - self.missing_property_service = CloudMissingPropertyService() + self.missing_property_service = OnPremiseMissingPropertyService() diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py index 1352adde..248ec92e 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py @@ -26,12 +26,14 @@ from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue from fiftyone_pipeline_core.jsonbundler import WKT_TYPES -from .clouddata import CloudData, CloudMissingPropertyService +from .clouddata import CloudData from .constants import Constants import json import threading +from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService + # Why a pipeline with a second CloudRequestEngine is refused. _ANOTHER_REQUEST_ENGINE = ( "This cloud engine is already in a pipeline with another Cloud Request " @@ -39,9 +41,11 @@ "pipeline with a different one is refused. Build another cloud engine " "for this pipeline, or use the same Cloud Request Engine in both.") -# The name the cloud missing property service had here, kept for code that -# imports it. -OnPremiseMissingPropertyService = CloudMissingPropertyService +class OnPremiseMissingPropertyService(MissingPropertyService): + + def check(self, key, flow_element): + + raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. Please check that the element and property names are correct.') class CloudEngine(Engine): @@ -128,20 +132,6 @@ def get_properties(self): self._load_properties(self._log) return super(CloudEngine, self).get_properties() - def properties_known(self): - - """! - Whether this engine has its properties from the CloudRequestEngine, - which it has once that engine's discovery has completed. The - properties are looked for first, as get_properties does. - - @rtype: bool - @return: True once the properties are known - """ - - self._load_properties(self._log) - return self._properties_final - def _load_properties(self, log): """! diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py index 28faf87e..fa02ff16 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py @@ -61,17 +61,16 @@ def get(self, key): @rtype mixed @return The value stored under the chosen key @raise Exception: where the property was left out of the engine's - restricted properties, or the data has no value for it, with the - message the missing property service gives for the reason + restricted properties, saying so, or where the data has no value + for it, with the message the missing property service gives """ if hasattr(self.flow_element, "restricted_property_keys"): if key.lower() not in self.flow_element.restricted_property_keys: - # The service gives the reason, as the property was - # excluded or is in a data tier the data file lacks. - return self.missing_property_service.check( - key, self.flow_element) + raise Exception( + "Property " + key + " was excluded from " + + self.flow_element.datakey) try: if self.get_internal(key) == None: diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/missingproperty_service.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/missingproperty_service.py index 35c4a7c6..c99faa00 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/missingproperty_service.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/missingproperty_service.py @@ -20,104 +20,13 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -from enum import Enum - - -class MissingPropertyReason(Enum): - """! - Why a property an engine defines is missing from its data, named as the - .NET packages name each reason. - """ - - DATA_FILE_UPGRADE_REQUIRED = "DataFileUpgradeRequired" - PROPERTY_EXCLUDED_FROM_ENGINE_CONFIGURATION = \ - "PropertyExcludedFromEngineConfiguration" - PRODUCT_NOT_ACCESSIBLE_WITH_RESOURCE_KEY = \ - "ProductNotAccessibleWithResourceKey" - PROPERTY_NOT_ACCESSIBLE_WITH_RESOURCE_KEY = \ - "PropertyNotAccessibleWithResourceKey" - CLOUD_REQUEST_FAILED = "CloudRequestFailed" - UNKNOWN = "Unknown" - - -# The pages the messages link to, with this repository's campaign -# parameters. -PRICING_URL = "https://51degrees.com/pricing?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-pricing" -CONFIGURATOR_URL = "https://configure.51degrees.com/?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-configurator" - -# Lists in a message are joined with a comma and no space. -LIST_SEPARATOR = "," - - -def missing_property_message( - reason, name, element, tiers=(), engine_type="", properties=()): - """! - The message of the missing properties table for a reason. - - @type reason: MissingPropertyReason - @param reason: why the property is missing - @type name: str - @param name: the property, as the caller named it - @type element: str - @param element: the data key of the element asked - @type tiers: list - @param tiers: the data tiers that hold the property, for - DATA_FILE_UPGRADE_REQUIRED, where an empty list gives the row for - tiers that are not known - @type engine_type: str - @param engine_type: the engine's type name, for - DATA_FILE_UPGRADE_REQUIRED - @type properties: list - @param properties: the property names the key includes, for - PROPERTY_NOT_ACCESSIBLE_WITH_RESOURCE_KEY - @rtype: str - @return: the message - """ - - text = ("Property '" + name + "' not found in data for element '" + - element + "'. ") - if reason is MissingPropertyReason.DATA_FILE_UPGRADE_REQUIRED: - text += ("This is because your license or data file does not " - "include this property. ") - if tiers: - return text + ( - "It is available with the " + LIST_SEPARATOR.join(tiers) + - " license/data for the " + engine_type + - ". To upgrade, see " + PRICING_URL + ".") - return text + ( - "To see which licenses include it, see " + PRICING_URL + ".") - if reason is MissingPropertyReason.PROPERTY_EXCLUDED_FROM_ENGINE_CONFIGURATION: - return text + ("This is because the property has been excluded " - "when configuring the engine.") - if reason is MissingPropertyReason.PRODUCT_NOT_ACCESSIBLE_WITH_RESOURCE_KEY: - return text + ( - "This is because your resource key does not include access to " - "any properties under '" + element + "'. Check or create a " - "resource key with the properties you need in the configurator " - "at " + CONFIGURATOR_URL + ".") - if reason is MissingPropertyReason.PROPERTY_NOT_ACCESSIBLE_WITH_RESOURCE_KEY: - return text + ( - "This is because your resource key does not include access to " - "this property. The properties included for your key under '" + - element + "' are " + LIST_SEPARATOR.join(properties) + - ". Check or change your key in the configurator at " + - CONFIGURATOR_URL + ".") - if reason is MissingPropertyReason.CLOUD_REQUEST_FAILED: - return text + ( - "This is because an upstream cloud request failed, so no data " - "was available to populate the property.") - return text + ("The reason for this is unknown. Please check that the " - "aspect and property name are correct.") - class MissingPropertyService(): """! - Raises the error for a property an engine's data does not hold, giving - the reason in the order the properties specification sets out, with - the missing properties table's message for it. Property names are - compared ignoring case. Engines of a kind override needs_upgrade and - engine_reason to add their own steps. + A missing property service runs when a property is not available in the + aspectData. It can be extended to provide a specific message for why the property + is not available """ def check(self, key, flow_element): @@ -129,121 +38,7 @@ def check(self, key, flow_element): @param key: the property asked for @type flow_element: FlowElement @param flow_element: the element whose data was asked - @raise Exception: always, with the message for the reason the - property is missing - """ - - raise Exception(self.message(key, flow_element)) - - def message(self, key, flow_element): - - """! - The message for the reason a property is missing. - - @type key: string - @param key: the property asked for - @type flow_element: FlowElement - @param flow_element: the element whose data was asked - @rtype: str - @return: the message - """ - - name = key.lower() - listed = None - for property_name, meta in self.properties(flow_element).items(): - if property_name.lower() == name: - listed = meta - break - restricted = getattr(flow_element, "restricted_property_keys", None) - - # A property the metadata lists, or one the restricted list leaves - # out, is in a data tier the data file does not hold, or was - # excluded. - if listed is not None or ( - restricted is not None and name not in restricted): - tiers = list((listed or {}).get("datafiles") or ()) - if listed is not None and self.needs_upgrade(tiers, flow_element): - return missing_property_message( - MissingPropertyReason.DATA_FILE_UPGRADE_REQUIRED, key, - flow_element.datakey, tiers=tiers, - engine_type=type(flow_element).__name__) - return missing_property_message( - MissingPropertyReason.PROPERTY_EXCLUDED_FROM_ENGINE_CONFIGURATION, - key, flow_element.datakey) - - reason, properties = self.engine_reason(flow_element) - return missing_property_message( - reason, key, flow_element.datakey, properties=properties) - - def properties(self, flow_element): - - """! - The engine's property metadata, by name. - - @type flow_element: FlowElement - @param flow_element: the engine - @rtype: dict - @return: the metadata of each property, by name - """ - - get_properties = getattr(flow_element, "get_properties", None) - if get_properties is not None: - return get_properties() or {} - return getattr(flow_element, "properties", None) or {} - - def needs_upgrade(self, tiers, flow_element): - - """! - Whether a property the metadata lists is missing because the - engine's data file is not of one of the property's tiers. An engine - with no data file never needs an upgrade. - - @type tiers: list - @param tiers: the data tiers that hold the property - @type flow_element: FlowElement - @param flow_element: the engine - @rtype: bool - @return: True where an upgrade is needed - """ - - return False - - def engine_reason(self, flow_element): - - """! - The reason for a property the metadata does not list, which a kind - of engine with a step of its own overrides. - - @type flow_element: FlowElement - @param flow_element: the engine - @rtype: tuple - @return: the reason, and the property names for a message that - lists them - """ - - return MissingPropertyReason.UNKNOWN, () - - -class OnPremiseMissingPropertyService(MissingPropertyService): - - """! - The missing property service of an on-premise engine, whose data file - tier decides whether a property the metadata lists needs an upgrade. - The engine's native engine gives the tier. - """ - - def needs_upgrade(self, tiers, flow_element): - - """! - Whether the tier of the data file the engine uses is not among a - property's tiers. - - @type tiers: list - @param tiers: the data tiers that hold the property - @type flow_element: FlowElement - @param flow_element: the engine - @rtype: bool - @return: True where an upgrade is needed + @raise Exception: always, saying why the property is missing """ - return flow_element.engine.getProduct() not in tiers + raise Exception("Property " + key + " not found in " + flow_element.datakey) From c781d953e5d69c3e891ada92cbf21fe57d101a76 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 21:26:39 +0000 Subject: [PATCH 3/7] Share every query parameter for a list of None when the option is not given ShareUsageEvidenceKeyFilter given query_whitelist=None, with share_all_query_string_parameters not given, shares every query string and form parameter, as the released filter did, because no sentence of the specification says how None is read. The option now defaults to None for not given. Given as False, the specification's MUST holds, so only parameters starting 51D_ and those on the list are shared, and given as True every parameter is. The ShareUsage element always gives the option, so it shares as before. --- .../share_usage_evidencekeyfilter.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py index 9438ff5a..b466fd13 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py @@ -67,14 +67,17 @@ def __init__( header_blacklist = (), include_session = False, share_all = False, - share_all_query_string_parameters = False): + share_all_query_string_parameters = None): """! Constructor for ShareUsageEvidenceKeyFilter @type cookie: string @param cookie: the name of the cookie that identifies the session @type query_whitelist: list or str @param query_whitelist: query string and form parameters to share as - well as those starting 51D_. None holds no names. A string is read as names separated by commas. + well as those starting 51D_. A string is read as names separated by + commas. None shares every parameter when + share_all_query_string_parameters is not given, and holds no names + when it is. @type header_blacklist: list or str @param header_blacklist: headers not to share. A string is read as names separated by commas. @@ -85,13 +88,20 @@ def __init__( evidence that is never shared and cookies not starting 51D_ @type share_all_query_string_parameters: bool @param share_all_query_string_parameters: whether every query - string and form parameter is shared, whatever query_whitelist holds + string and form parameter is shared, whatever query_whitelist holds. + When it is not given, every one is shared only when query_whitelist + is None. """ self.cookie = cookie.lower() if cookie else None query_whitelist = _split_names(query_whitelist) header_blacklist = _split_names(header_blacklist) + # With the option not given, None in place of the list asks for + # every parameter. Given, the option decides. + if share_all_query_string_parameters is None: + share_all_query_string_parameters = query_whitelist is None + self.query_whitelist = \ [parameter.lower() for parameter in query_whitelist or ()] From 28fcabae664abd4c1c517ca54bcc6df9bee4ee78 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 18 Nov 2026 01:56:02 +0000 Subject: [PATCH 4/7] Test that only 51D_ parameters and those the Pipeline uses are shared Query string and form parameters are shared only when their names start with 51D_ or a Flow Element of the Pipeline accepts them as evidence, from a GET query string and a POST form body alike, whatever the settings. The deprecated query_whitelist and share_all_query_string_parameters options share nothing more, and the JavaScript Builder counts only for the parameters its specification lists. The filter, list and usage tests that relied on those options, or on every parameter being shared, are updated to the rule. The released test test_default_shares_only_accepted_evidence goes back to its released form. --- .../test_share_usage_evidencekeyfilter.py | 131 ++++++++++-------- .../tests/test_share_usage_lists.py | 26 ++-- .../tests/test_share_usage_rules.py | 129 +++++++++++------ .../tests/test_usage.py | 38 +++-- 4 files changed, 204 insertions(+), 120 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py index f94a4d90..25c0087e 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py @@ -22,10 +22,31 @@ import unittest +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.flowelement import FlowElement from fiftyone_pipeline_engines_fiftyone.share_usage_evidencekeyfilter import \ ShareUsageEvidenceKeyFilter +class UsingElement(FlowElement): + """Accepts the evidence keys given.""" + + def __init__(self, keys): + super(UsingElement, self).__init__() + self.keys = keys + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter(self.keys) + + +class UsingPipeline: + """Stands in for a Pipeline whose FlowElement uses the keys given.""" + + def __init__(self, *keys): + self.flow_elements = [UsingElement(list(keys))] + + class ShareUsageEvidenceKeyFilterTests(unittest.TestCase): def test_default_rules(self): @@ -53,20 +74,22 @@ def test_default_rules(self): def test_field_is_everything_after_the_first_dot(self): """! - Check that a name containing a dot is matched as a whole against - the parameters to share. + Check that a parameter whose name contains a dot is matched as a + whole against the evidence the Pipeline uses. """ - keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"]) + keys = ShareUsageEvidenceKeyFilter() + pipeline = UsingPipeline("query.valid.email") - self.assertTrue(keys.filter_evidence_key("query.valid.email")) - self.assertFalse(keys.filter_evidence_key("query.valid")) - self.assertFalse(keys.filter_evidence_key("query.other.name")) + self.assertTrue(keys.filter_evidence_key("query.valid.email", pipeline)) + self.assertFalse(keys.filter_evidence_key("query.valid", pipeline)) + self.assertFalse(keys.filter_evidence_key("query.other.name", pipeline)) def test_lists_ignore_case(self): """! - Check that the headers not to share and the parameters to share - are matched without regard to case. + Check that the headers not to share are matched without regard to + case, and that a parameter on the deprecated list to share is not + shared. """ keys = ShareUsageEvidenceKeyFilter( @@ -74,60 +97,55 @@ def test_lists_ignore_case(self): header_blacklist=["X-Secret"]) self.assertFalse(keys.filter_evidence_key("header.x-secret")) - self.assertTrue(keys.filter_evidence_key("query.campaign")) + self.assertFalse(keys.filter_evidence_key("query.campaign")) self.assertTrue(keys.filter_evidence_key("header.user-agent")) def test_lists_from_comma_separated_strings(self): """! Check that a string given for either list, the form a configuration file gives a list in, is read as names separated by - commas, each trimmed. + commas, each trimmed, and that the parameters named still are not + shared. """ keys = ShareUsageEvidenceKeyFilter( query_whitelist="Campaign, utm_source", header_blacklist="X-Secret,X-Other") - self.assertTrue(keys.filter_evidence_key("query.campaign")) - self.assertTrue(keys.filter_evidence_key("query.utm_source")) - self.assertFalse(keys.filter_evidence_key("query.c")) + self.assertEqual(["campaign", "utm_source"], keys.query_whitelist) + self.assertFalse(keys.filter_evidence_key("query.campaign")) + self.assertFalse(keys.filter_evidence_key("query.utm_source")) self.assertFalse(keys.filter_evidence_key("header.x-secret")) self.assertFalse(keys.filter_evidence_key("header.x-other")) self.assertTrue(keys.filter_evidence_key("header.x")) def test_no_parameter_list_shares_every_parameter(self): """! - Check that None in place of the parameters to share shares every - parameter, and an empty list only those starting 51D_. + Check that None in place of the deprecated parameters to share, and + an empty list, both share only parameters starting 51D_ when the + Pipeline uses no others. """ - every = ShareUsageEvidenceKeyFilter(query_whitelist=None) - prefixed = ShareUsageEvidenceKeyFilter(query_whitelist=[]) + for query_whitelist in (None, []): + keys = ShareUsageEvidenceKeyFilter(query_whitelist=query_whitelist) - self.assertTrue(every.filter_evidence_key("query.campaign")) - self.assertFalse(prefixed.filter_evidence_key("query.campaign")) - self.assertTrue(prefixed.filter_evidence_key("query.51d_campaign")) + self.assertFalse(keys.filter_evidence_key("query.campaign")) + self.assertTrue(keys.filter_evidence_key("query.51d_campaign")) def test_the_option_given_decides_whatever_the_list(self): """! - Check that with share_all_query_string_parameters given, None in - place of the parameters to share holds no names, as an empty list - does, so the option off shares only parameters starting 51D_ and - the option on shares every parameter. + Check that the deprecated share_all_query_string_parameters option + shares nothing more, on or off, whatever the list. """ - for query_whitelist in (None, []): - prefixed = ShareUsageEvidenceKeyFilter( - query_whitelist=query_whitelist, - share_all_query_string_parameters=False) - every = ShareUsageEvidenceKeyFilter( - query_whitelist=query_whitelist, - share_all_query_string_parameters=True) - - self.assertFalse(prefixed.filter_evidence_key("query.campaign")) - self.assertTrue( - prefixed.filter_evidence_key("query.51d_campaign")) - self.assertTrue(every.filter_evidence_key("query.campaign")) + for query_whitelist in (None, [], ["campaign"]): + for share_every in (False, True): + keys = ShareUsageEvidenceKeyFilter( + query_whitelist=query_whitelist, + share_all_query_string_parameters=share_every) + + self.assertFalse(keys.filter_evidence_key("query.campaign")) + self.assertTrue(keys.filter_evidence_key("query.51d_campaign")) def test_session_cookie(self): """! @@ -146,8 +164,9 @@ def test_session_cookie(self): def test_share_all(self): """! Check that sharing everything shares the evidence the other rules - leave out, apart from the evidence that is never shared and the - cookies that do not start 51D_. + leave out, apart from the evidence that is never shared, the + cookies that do not start 51D_, and the parameters that neither start + 51D_ nor are used by the Pipeline. """ keys = ShareUsageEvidenceKeyFilter(share_all=True) @@ -156,8 +175,9 @@ def test_share_all(self): self.assertFalse(keys.filter_evidence_key("cookie.sessionid")) self.assertTrue( keys.filter_evidence_key("cookie.51D_screenpixelswidth")) - self.assertTrue(keys.filter_evidence_key("query.password")) - self.assertTrue(keys.filter_evidence_key("query.id.usage")) + self.assertFalse(keys.filter_evidence_key("query.password")) + self.assertFalse(keys.filter_evidence_key("query.id.usage")) + self.assertTrue(keys.filter_evidence_key("query.51d_usage")) self.assertTrue(keys.filter_evidence_key("server.request-method")) self.assertFalse(keys.filter_evidence_key("query.id.email")) self.assertFalse(keys.filter_evidence_key("header.id.email")) @@ -168,18 +188,18 @@ def test_share_all(self): def test_every_query_parameter_when_sharing_them_all(self): """! - Check that sharing every query string parameter shares one not on - the list, and that a filter made without it keeps to the list. + Check that a parameter is shared when the Pipeline uses it, and + another is not, whatever the deprecated options say. """ every = ShareUsageEvidenceKeyFilter( - query_whitelist=["campaign"], - share_all_query_string_parameters=True) - listed = ShareUsageEvidenceKeyFilter(query_whitelist=["campaign"]) + query_whitelist=["password"], + share_all_query_string_parameters=True, + share_all=True) + pipeline = UsingPipeline("query.campaign") - self.assertTrue(every.filter_evidence_key("query.password")) - self.assertFalse(listed.filter_evidence_key("query.password")) - self.assertTrue(listed.filter_evidence_key("query.campaign")) + self.assertFalse(every.filter_evidence_key("query.password", pipeline)) + self.assertTrue(every.filter_evidence_key("query.campaign", pipeline)) def test_the_session_cookie_is_kept_for_the_tracker_alone(self): """! @@ -222,12 +242,13 @@ def test_never_shared_keys_match_whole_parts(self): never shared is treated as an ordinary key. """ - keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"]) + keys = ShareUsageEvidenceKeyFilter() - self.assertTrue(keys.filter_evidence_key("query.valid.email")) + self.assertTrue(keys.filter_evidence_key( + "query.valid.email", UsingPipeline("query.valid.email"))) self.assertTrue( ShareUsageEvidenceKeyFilter(share_all=True) - .filter_evidence_key("query.valid.email")) + .filter_evidence_key("header.valid.email")) def test_naming_a_parameter_does_not_share_the_query_string(self): """! @@ -235,7 +256,9 @@ def test_naming_a_parameter_does_not_share_the_query_string(self): string, which carries every parameter. """ - keys = ShareUsageEvidenceKeyFilter(query_whitelist=["utm_source"]) + keys = ShareUsageEvidenceKeyFilter() + pipeline = UsingPipeline("query.utm_source") - self.assertTrue(keys.filter_evidence_key("query.utm_source")) - self.assertFalse(keys.filter_evidence_key("server.request-query")) + self.assertTrue(keys.filter_evidence_key("query.utm_source", pipeline)) + self.assertFalse( + keys.filter_evidence_key("server.request-query", pipeline)) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py index 9209ae53..b31b9f1f 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py @@ -21,8 +21,9 @@ # ********************************************************************* """A configuration file gives a list of names as one comma-separated string, -so the usage sharing element takes its query string parameters to share and -its headers not to share in that form, as well as in a list.""" +so the usage sharing element takes its headers not to share in that form, as +well as in a list. Its deprecated query string parameters to share are read +the same way and shared no more than any other parameter.""" import json import unittest @@ -44,8 +45,7 @@ "endpoint": "http://127.0.0.1:9", } -# Shares only the query string parameters listed and those starting 51D_, -# where every one is shared by default. +# The deprecated option, which no longer changes what is shared. LISTED_ONLY = {"share_all_query_string_parameters": False} # A configuration file that gives both lists as comma-separated strings. @@ -114,8 +114,9 @@ def test_query_parameters_to_share_from_a_string(self): ShareUsage(query_whitelist="first,second", **LISTED_ONLY, **QUEUE_ONLY)) - self.assert_shared(shared, ("first value", "second value")) - self.assert_not_shared(shared, ("third value", "letter value")) + self.assert_not_shared( + shared, + ("first value", "second value", "third value", "letter value")) def test_headers_not_to_share_from_a_string(self): shared = shared_alone_by( @@ -130,10 +131,11 @@ def test_lists_from_a_configuration_file(self): usage = pipeline.get_element("shareusage") shared = shared_by(pipeline, usage) - self.assert_shared( - shared, ("first value", "second value", "agent value")) + self.assertEqual(["first", "second"], usage.query_whitelist) + self.assert_shared(shared, ("agent value",)) self.assert_not_shared( - shared, ("third value", "letter value", "one value", "two value")) + shared, ("first value", "second value", "third value", + "letter value", "one value", "two value")) def test_names_are_trimmed_and_empty_entries_left_out(self): usage = ShareUsage( @@ -165,10 +167,10 @@ def test_a_list_is_used_as_it_is(self): self.assertIs(names, usage.query_whitelist) self.assertIs(headers, usage.header_blacklist) shared = shared_alone_by(usage) - self.assert_shared( - shared, ("first value", "second value", "agent value")) + self.assert_shared(shared, ("agent value",)) self.assert_not_shared( - shared, ("third value", "letter value", "one value", "two value")) + shared, ("first value", "second value", "third value", + "letter value", "one value", "two value")) if __name__ == "__main__": diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py index 5df7e9be..40a4bd51 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py @@ -21,10 +21,11 @@ # ********************************************************************* """ -What usage sharing shares by default and when all Evidence is shared. -Every query string and form parameter is shared unless that is turned off, -no cookie but a 51D_ one is ever shared, and the session cookie is still -compared when sessions are tracked. +What usage sharing shares by default and when all Evidence is shared. A query +string or form parameter is shared only when its name starts with 51D_ or the +Pipeline uses it as evidence, whatever the settings, no cookie but a 51D_ one +is ever shared, and the session cookie is still compared when sessions are +tracked. """ import gzip @@ -36,6 +37,8 @@ import requests from flask import Flask, request +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter from fiftyone_pipeline_core.flowelement import FlowElement from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_pipeline_core.web import webevidence @@ -53,8 +56,8 @@ "endpoint": "http://127.0.0.1:9", } -# Turns off sharing every query string parameter, from a configuration -# file. +# The deprecated option to share every query string parameter, from a +# configuration file. CONFIGURATION = """ { "PipelineOptions": { @@ -104,50 +107,91 @@ def records_of(usage, requests_evidence, *elements): return sent_on_close(usage) -class QueryAndFormValuesTests(unittest.TestCase): +class UsingElement(FlowElement): + """Accepts the query string parameter given as evidence, and adds no + data.""" + + def __init__(self, key): + super(UsingElement, self).__init__() + self.datakey = "using" + self.key = key + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter([self.key]) + + +# Every way the deprecated options could once share more. +DEPRECATED_SETTINGS = ( + {}, + {"share_all": True}, + {"query_whitelist": ["password", "campaign"]}, + {"share_all_query_string_parameters": True}, + {"share_all_query_string_parameters": False, + "query_whitelist": "password,campaign"}, +) + + +def shared_from_web_request(settings, method): + """The record usage sharing sends for one web request with the + parameters below, in its query string for GET or its form body for + POST, through a Pipeline whose other element uses query.used.""" + app = Flask(__name__) + usage = ShareUsage(**dict(QUEUE_ONLY, **settings)) + pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS) \ + .add(UsingElement("query.used")).add(usage).build() + parameters = {"51D_screen": "fiftyone value", "used": "used value", + "password": "password value", "campaign": "campaign value"} + if method == "GET": + context = app.test_request_context("/", query_string=parameters) + else: + context = app.test_request_context( + "/", method="POST", data=parameters) + with context: + flowdata = pipeline.create_flowdata() + flowdata.evidence.add_from_dict(webevidence(request)) + flowdata.process() + records = sent_on_close(usage) + return records[0] if records else "" - def test_every_query_value_is_shared_by_default(self): - records = records_of(ShareUsage(**QUEUE_ONLY), [{ - "header.user-agent": "agent value", - "query.campaign": "campaign value", - "query.password": "password value", - }]) - self.assertEqual(1, len(records)) - self.assertIn('campaign value', - records[0]) - self.assertIn('password value', - records[0]) +class QueryAndFormValuesTests(unittest.TestCase): - def test_every_form_value_of_the_web_request_is_shared_by_default(self): - app = Flask(__name__) + def test_a_51d_parameter_is_shared_from_a_query_string_and_a_form(self): + for method in ("GET", "POST"): + with self.subTest(method=method): + record = shared_from_web_request({}, method) + self.assertIn("fiftyone value", record) + + def test_a_parameter_the_pipeline_uses_is_shared_from_both(self): + for method in ("GET", "POST"): + with self.subTest(method=method): + record = shared_from_web_request({}, method) + self.assertIn('used value', record) + + def test_no_other_parameter_is_ever_shared_from_either(self): + for settings in DEPRECATED_SETTINGS: + for method in ("GET", "POST"): + with self.subTest(settings=settings, method=method): + record = shared_from_web_request(settings, method) + self.assertIn("fiftyone value", record) + self.assertNotIn("password value", record) + self.assertNotIn("campaign value", record) + + def test_the_javascript_builder_uses_only_the_parameters_it_lists(self): usage = ShareUsage(**QUEUE_ONLY) - pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS).add(usage).build() - with app.test_request_context( - "/", method="POST", data={"field": "form value"}): - flowdata = pipeline.create_flowdata() - flowdata.evidence.add_from_dict(webevidence(request)) - flowdata.process() + pipeline = PipelineBuilder().add(usage).build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "agent value") + flowdata.evidence.add("query.password", "password value") + flowdata.evidence.add("query.fod-js-object-name", "object value") + flowdata.process() records = sent_on_close(usage) - self.assertEqual(1, len(records)) - self.assertIn('form value', records[0]) - - def test_turned_off_only_51d_and_listed_values_are_shared(self): - records = records_of(ShareUsage( - share_all_query_string_parameters=False, - query_whitelist=["campaign"], **QUEUE_ONLY), [{ - "header.user-agent": "agent value", - "query.campaign": "campaign value", - "query.51d_gethighentropyvalues": "hev", - "query.password": "password value", - }]) - - self.assertIn("campaign value", records[0]) - self.assertIn("hev", records[0]) + self.assertIn("agent value", records[0]) + self.assertIn("object value", records[0]) self.assertNotIn("password value", records[0]) - def test_turned_off_from_a_configuration_file(self): + def test_the_deprecated_option_from_a_configuration_file(self): pipeline = PipelineBuilder().build_from_configuration( json.loads(CONFIGURATION)) usage = pipeline.get_element("shareusage") @@ -161,7 +205,6 @@ def test_turned_off_from_a_configuration_file(self): self.assertIn("agent value", records[0]) self.assertNotIn("password value", records[0]) - class CookieTests(unittest.TestCase): def test_only_51d_cookies_are_shared_when_all_evidence_is_shared(self): diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index c415387e..c3749389 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -38,6 +38,22 @@ import requests from unittest import mock +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.flowelement import FlowElement + + +class UsingElement(FlowElement): + """Accepts the evidence keys given, and adds no data.""" + + def __init__(self, keys): + super(UsingElement, self).__init__() + self.datakey = "using" + self.keys = keys + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter(self.keys) + testHost = "127.0.0.1" testPort = 5000 testEndpoint = f"http://{testHost}:{testPort}" @@ -919,9 +935,9 @@ def test_escapes_xml_characters(self): def test_default_shares_only_accepted_evidence(self): """! - Check that with no options the element shares headers, the - 51Degrees cookies and every query string and form parameter, and - leaves out other cookies, the Cookie header and the email address. + Check that with no options the element shares headers and the + 51Degrees cookies and parameters, and leaves out other cookies, the + Cookie header, and other query string and form parameters. """ usage = ShareUsage( @@ -948,7 +964,7 @@ def test_default_shares_only_accepted_evidence(self): self.assertIn('hev', app.received[0]) self.assertNotIn("cookie value", app.received[0]) self.assertNotIn("header value", app.received[0]) - self.assertIn('password value', app.received[0]) + self.assertNotIn("password value", app.received[0]) self.assertNotIn("email value", app.received[0]) def test_never_shared_evidence(self): @@ -984,8 +1000,9 @@ def test_never_shared_evidence(self): def test_share_all(self): """! Check that sharing all evidence shares what the other rules leave - out, but not the evidence that is never shared or a cookie that - does not start 51D_. + out, but not the evidence that is never shared, a cookie that + does not start 51D_, or a parameter that neither starts 51D_ nor is + used by the Pipeline. """ usage = ShareUsage( @@ -1007,22 +1024,21 @@ def test_share_all(self): self.assertEqual(len(app.received), 1) self.assertNotIn("cookie value", app.received[0]) self.assertIn('1080', app.received[0]) - self.assertIn('campaign value', app.received[0]) + self.assertNotIn("campaign value", app.received[0]) self.assertNotIn("email value", app.received[0]) def test_parameter_names_with_dots(self): """! Check that a query string parameter whose name contains a dot is - matched by its whole name against the parameters to share. + matched by its whole name against the evidence the Pipeline uses. """ usage = ShareUsage( requested_package_size=1, - endpoint=testEndpoint, - share_all_query_string_parameters=False, - query_whitelist=["valid.email"]) + endpoint=testEndpoint) pipeline = (PipelineBuilder())\ + .add(UsingElement(["query.valid.email"]))\ .add(usage)\ .build() flowdata = pipeline.create_flowdata() From 715e618b1d23ab61d27347fde5c1b06b8b5a3d61 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 18 Nov 2026 01:56:03 +0000 Subject: [PATCH 5/7] Share a parameter only when it starts with 51D_ or the Pipeline uses it The usage sharing filter decides a query. key, which carries query string and form parameters alike, before share_all and whatever the configuration. It is shared when its name starts with 51D_ or a Flow Element of the Pipeline accepts it as evidence, so a value such as a password is never sent. The JavaScript Builder passes every parameter so that it can send them back to its callback, so it is asked only about the two parameters its specification lists. query_whitelist and share_all_query_string_parameters are deprecated and have no effect, and are still accepted so that existing code and configuration build. --- .../share_usage.py | 26 ++--- .../share_usage_evidencekeyfilter.py | 95 +++++++++++++------ 2 files changed, 81 insertions(+), 40 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 88cb2f61..74ca6f9c 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -144,11 +144,10 @@ def __init__( needs cookie to be set, and works the same whether or not share_all is True. @type query_whitelist: list or str - @param query_whitelist: The query string and HTTP form parameters - to share, as well as those prefixed with '51D_', when - share_all_query_string_parameters is False. A string of names - separated by commas, which is how a configuration file gives a - list, is read as those names. None holds no names. + @param query_whitelist: Deprecated, and has no effect. A query + string or HTTP form parameter is shared only when its name starts + with '51D_' or the Pipeline uses it as evidence, whatever the + settings. @type header_blacklist: list or str @param header_blacklist: By default, all HTTP headers except Cookie are shared. @@ -160,13 +159,13 @@ def __init__( @type share_all: bool @param share_all: If True, all evidence is shared apart from the email address, the salt used for hashed email identifiers, the - request's query string and path, and every cookie whose name does - not start with '51D_'. False by default. + request's query string and path, every cookie whose name does not + start with '51D_', and every query string or HTTP form parameter + whose name does not start with '51D_' and that the Pipeline does + not use as evidence. False by default. @type share_all_query_string_parameters: bool - @param share_all_query_string_parameters: If True, every query - string and HTTP form parameter is shared. If False, only those - prefixed with '51D_' and those in query_whitelist are. True by - default. + @param share_all_query_string_parameters: Deprecated, and has no + effect, as query_whitelist. @type max_queue_size: int @param max_queue_size: The most entries held while waiting to be sent. When the queue is full, new entries are dropped. The default @@ -741,7 +740,7 @@ def process_internal(self, flow_data): # A hash of the values that would be shared, as the element # accepts no evidence of its own. cachekey = self.tracker_evidence_key_filter.filter_evidence( - self.found_evidence(flow_data)) + self.found_evidence(flow_data), flow_data.pipeline) # Evidence keys ignore case, so a repeat is found whatever the # letters of its keys. cachekey = json.dumps( @@ -784,7 +783,8 @@ def get_data_from_evidence(self, flow_data): f"The value '{value}' could not be parsed to an integer.") else: # Check if we can send this piece of evidence - if self.data_evidence_key_filter.filter_evidence_key(key): + if self.data_evidence_key_filter.filter_evidence_key( + key, flow_data.pipeline): data.try_add(key, value) return data diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py index b466fd13..3647879a 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py @@ -21,6 +21,7 @@ # ********************************************************************* from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.javascriptbuilder import JavascriptBuilderElement # Cookies, session values and query string parameters whose names start with # this are the ones 51Degrees sets, and are shared. @@ -37,6 +38,30 @@ "id.email", "id.salt", "request-query", "request-path", "header.cookie") +# The query string parameters the JavaScript Builder's specification lists as +# its accepted Evidence. Its filter also passes every other parameter, only so +# that it can send them back to the callback. +JAVASCRIPT_BUILDER_PARAMETERS = ( + "query.fod-js-object-name", "query.fod-js-enable-cookies") + + +def _used_by(pipeline, key): + """! + Whether a FlowElement of the Pipeline accepts the key as evidence. The + JavaScript Builder is asked only about the parameters its specification + says it accepts. + """ + if pipeline is None: + return False + for element in pipeline.flow_elements: + if isinstance(element, JavascriptBuilderElement): + if key.lower() in JAVASCRIPT_BUILDER_PARAMETERS: + return True + elif element.get_evidence_key_filter().filter_evidence_key(key): + return True + return False + + def _split_names(names): """! A configuration file gives a list of names as one comma-separated @@ -53,10 +78,10 @@ class ShareUsageEvidenceKeyFilter(BasicListEvidenceKeyFilter): """ The ShareUsageEvidenceKeyFilter decides which evidence the 51Degrees ShareUsage service is sent. Cookies are shared only when their names - start with 51D_, whatever the other settings. Headers are shared unless - they are blocked, session values only when their names start with 51D_, - and query string and form parameters when all of them are shared or - when their names start with 51D_ or are on the list to share. Any other + start with 51D_, and query string and form parameters only when their + names start with 51D_ or the Pipeline uses them as evidence, whatever + the other settings. Headers are shared unless they are blocked, and + session values only when their names start with 51D_. Any other evidence is shared. """ @@ -73,11 +98,9 @@ def __init__( @type cookie: string @param cookie: the name of the cookie that identifies the session @type query_whitelist: list or str - @param query_whitelist: query string and form parameters to share as - well as those starting 51D_. A string is read as names separated by - commas. None shares every parameter when - share_all_query_string_parameters is not given, and holds no names - when it is. + @param query_whitelist: deprecated, and has no effect. A query + string or form parameter is shared only when its name starts with + 51D_ or the Pipeline uses it as evidence. @type header_blacklist: list or str @param header_blacklist: headers not to share. A string is read as names separated by commas. @@ -87,21 +110,16 @@ def __init__( @param share_all: whether all evidence is shared, apart from the evidence that is never shared and cookies not starting 51D_ @type share_all_query_string_parameters: bool - @param share_all_query_string_parameters: whether every query - string and form parameter is shared, whatever query_whitelist holds. - When it is not given, every one is shared only when query_whitelist - is None. + @param share_all_query_string_parameters: deprecated, and has no + effect, as query_whitelist. """ self.cookie = cookie.lower() if cookie else None query_whitelist = _split_names(query_whitelist) header_blacklist = _split_names(header_blacklist) - # With the option not given, None in place of the list asks for - # every parameter. Given, the option decides. - if share_all_query_string_parameters is None: - share_all_query_string_parameters = query_whitelist is None - + # Kept so that code reading them still works. Neither decides what + # is shared. self.query_whitelist = \ [parameter.lower() for parameter in query_whitelist or ()] @@ -115,18 +133,40 @@ def __init__( self.share_all_query_string_parameters = \ share_all_query_string_parameters - def filter_evidence_key(self, key): + def filter_evidence(self, evidence_key_object, pipeline=None): + + """! + The evidence to share, from a dictionary of evidence. + + @type evidence_key_object: dict + @param evidence_key_object: evidence keys and values + @type pipeline: Pipeline + @param pipeline: the Pipeline the evidence came to, whose evidence + decides which query string and form parameters are shared + @rtype: dict + @return: the evidence to share + """ + + return {key: value for key, value in evidence_key_object.items() + if self.filter_evidence_key(key, pipeline)} + + def filter_evidence_key(self, key, pipeline=None): """! Check if a specific key should be filtered. @type key: string @param key: to check in the filter + @type pipeline: Pipeline + @param pipeline: the Pipeline the evidence came to. Without one, a + query string or form parameter is shared only when its name starts + with 51D_. @rtype: bool @return: Is this key in the filter's keys list? """ + original = key key = key.lower() if any(key == never_shared or key.endswith("." + never_shared) @@ -143,6 +183,13 @@ def filter_evidence_key(self, key): return field.startswith(FIFTYONE_PREFIX) or \ (self.include_session and field == self.cookie) + # A query string or form parameter is shared only when 51Degrees set + # it or the Pipeline uses it as evidence, even when all evidence is, + # so that a value such as a password is never sent. + if separator and prefix == "query": + return field.startswith(FIFTYONE_PREFIX) or \ + _used_by(pipeline, original) + if self.share_all: return True @@ -156,12 +203,6 @@ def filter_evidence_key(self, key): # Only session values that start with the 51Degrees # cookie prefix return field.startswith(FIFTYONE_PREFIX) - elif prefix == "query": - # Share every parameter only when asked to. Otherwise only - # those starting 51d_ or in the list. - return self.share_all_query_string_parameters or \ - field.startswith(FIFTYONE_PREFIX) or \ - field in self.query_whitelist else: # Add anything that is not a cookie, header, session # variable or query parameter @@ -174,10 +215,10 @@ class ShareUsageTrackerEvidenceKeyFilter(ShareUsageEvidenceKeyFilter): out, because they make every request different. """ - def filter_evidence_key(self, key): + def filter_evidence_key(self, key, pipeline=None): if key.lower() in ("query.session-id", "query.sequence"): return False return super(ShareUsageTrackerEvidenceKeyFilter, self)\ - .filter_evidence_key(key) + .filter_evidence_key(key, pipeline) From dc6e95ed1364c5b910df02c185a0e299171bbfb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eddie=20Vereb=C3=A9lyi?= Date: Wed, 18 Nov 2026 22:38:49 +0000 Subject: [PATCH 6/7] Name two parameter tests for what they now check test_no_parameter_list_shares_every_parameter checks that a list of None, or an empty one, shares only the parameters starting 51D_. It is now test_no_parameter_list_shares_only_51d_parameters. test_the_option_given_decides_whatever_the_list checks that the deprecated share_all_query_string_parameters option, on or off, changes nothing that is shared. It is now test_the_deprecated_option_changes_nothing. Only the names change, and both tests pass as before. --- .../tests/test_share_usage_evidencekeyfilter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py index 25c0087e..eb3a0fdc 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py @@ -119,7 +119,7 @@ def test_lists_from_comma_separated_strings(self): self.assertFalse(keys.filter_evidence_key("header.x-other")) self.assertTrue(keys.filter_evidence_key("header.x")) - def test_no_parameter_list_shares_every_parameter(self): + def test_no_parameter_list_shares_only_51d_parameters(self): """! Check that None in place of the deprecated parameters to share, and an empty list, both share only parameters starting 51D_ when the @@ -132,7 +132,7 @@ def test_no_parameter_list_shares_every_parameter(self): self.assertFalse(keys.filter_evidence_key("query.campaign")) self.assertTrue(keys.filter_evidence_key("query.51d_campaign")) - def test_the_option_given_decides_whatever_the_list(self): + def test_the_deprecated_option_changes_nothing(self): """! Check that the deprecated share_all_query_string_parameters option shares nothing more, on or off, whatever the list. From 1504f3edf58e0f556cda328fdacaefb5a86df0f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eddie=20Vereb=C3=A9lyi?= Date: Wed, 18 Nov 2026 22:40:37 +0000 Subject: [PATCH 7/7] Default the deprecated parameter option to False ShareUsageEvidenceKeyFilter took None for share_all_query_string_parameters to mean that the option was not given, so that a parameter list of None could share every parameter. The option now has no effect, so None means nothing and the default is False again, as the filter had on rc. Which evidence is shared does not change. --- .../share_usage_evidencekeyfilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py index 3647879a..e37fc724 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py @@ -92,7 +92,7 @@ def __init__( header_blacklist = (), include_session = False, share_all = False, - share_all_query_string_parameters = None): + share_all_query_string_parameters = False): """! Constructor for ShareUsageEvidenceKeyFilter @type cookie: string