diff --git a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py
index a19b7da2..b6e6fe5e 100644
--- a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py
+++ b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py
@@ -23,7 +23,7 @@
from __future__ import absolute_import
from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine
from fiftyone_pipeline_cloudrequestengine.clouddata import \
- CloudMissingPropertyService
+ OnPremiseMissingPropertyService
from fiftyone_pipeline_engines.aspectdata import AspectData
from fiftyone_pipeline_engines.aspectdata_dictionary import AspectDataDictionary
@@ -91,7 +91,7 @@ def __init__(self, flow_element, contents):
# The device data's service, so a name that is not a property is
# reported in the same way for both.
- self.missing_property_service = CloudMissingPropertyService()
+ self.missing_property_service = OnPremiseMissingPropertyService()
def _matching_key(self, key):
"""!
diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py
index 702e5c04..7d769b84 100644
--- a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py
+++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py
@@ -30,8 +30,6 @@
from fiftyone_devicedetection_cloud.devicedetection_cloud import DeviceDetectionCloud
-from fiftyone_pipeline_engines.missingproperty_service import CONFIGURATOR_URL
-
from .live_cloud import find_resource_key, no_resource_key
pytestmark = pytest.mark.live
@@ -158,16 +156,10 @@ def test_missing_property_service_not_found_anywhere(self):
fd.device.get("notpresent")
result = str(raised.exception)
- properties = pipeline.get_element("device").get_properties()
- self.assertIn(
- "Property 'notpresent' not found in data for element 'device'. "
- "This is because your resource key does not include access to "
- "this property. The properties included for your key under "
- "'device' are ", result)
- self.assertIn(
- ",".join(meta.get("name", name)
- for name, meta in properties.items()), result)
- self.assertIn(CONFIGURATOR_URL, result)
+ self.assertIn("notpresent", result)
+ self.assertIn("device", result)
+ self.assertIn(', '.join(pipeline.get_element("device").get_properties().keys()), result)
+ self.assertIn("https://51degrees.com/documentation/_services__cloud__resource_keys.html", result)
def test_cloud_request_origin(self):
"""!
diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py
index 05f4cc0f..7d7891fa 100644
--- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py
+++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py
@@ -25,8 +25,7 @@
from fiftyone_pipeline_engines.aspectdata import AspectData
from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue
-from fiftyone_pipeline_engines.missingproperty_service import \
- OnPremiseMissingPropertyService
+from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService
from .constants import RESULTS_CLOSED
@@ -34,6 +33,22 @@
# fiftyoneDegreesHashMatchMethod values, which getMethod returns.
MATCH_METHODS = ("NONE", "PERFORMANCE", "COMBINED", "PREDICTIVE")
+class OnPremiseMissingPropertyService(MissingPropertyService):
+
+ def check(self, key, flow_element):
+
+ if key in flow_element.properties:
+ datafiles = flow_element.properties[key]["datafiles"]
+ if(not flow_element.engine.getProduct() in datafiles):
+ raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. This is because your datafile does not contain the property. The property is available in' + str(datafiles))
+ # The data file holds the property, so it is missing because
+ # it is not available. A missing property always raises.
+ raise Exception(
+ "Property '" + key + "' not found in data for element '" +
+ flow_element.datakey + "'. This is because the property "
+ "has been excluded when configuring the engine.")
+ else:
+ raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. Please check that the element and property names are correct.')
class SwigData(AspectData):
diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py
index aa6222ca..f8151eaf 100644
--- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py
+++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py
@@ -261,9 +261,7 @@ def test_missing_property_service_not_found_anywhere(self):
result = str(e)
self.assertEqual(
- "Property 'notpresent' not found in data for element 'device'. "
- "The reason for this is unknown. Please check that the aspect and "
- "property name are correct.", result)
+ result, "Property notpresent not found in data for element device. Please check that the element and property names are correct.")
def test_missing_property_service_not_found_in_current(self):
"""!
@@ -288,20 +286,15 @@ def test_missing_property_service_not_found_in_current(self):
fd.process()
- # The name is found however it is spelled, and the message gives it
- # as the caller did.
- for name in ("mock", "Mock"):
- with self.assertRaises(Exception) as raised:
- fd.device.get(name)
- self.assertEqual(
- "Property '" + name + "' not found in data for element "
- "'device'. This is because your license or data file does not "
- "include this property. It is available with the Enterprise "
- "license/data for the DeviceDetectionOnPremise. To upgrade, "
- "see "
- "https://51degrees.com/pricing?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-pricing"
- ".",
- str(raised.exception))
+ result = ""
+
+ try:
+ fd.device.get("mock")
+ except Exception as e:
+ result = str(e)
+
+ self.assertEqual(
+ result, "Property mock not found in data for element device. This is because your datafile does not contain the property. The property is available in['Enterprise']")
def test_excluded_property(self):
"""!
@@ -329,9 +322,7 @@ def test_excluded_property(self):
result = str(e)
self.assertEqual(
- "Property 'screenpixelsheight' not found in data for element "
- "'device'. This is because the property has been excluded when "
- "configuring the engine.", result)
+ result, "Property screenpixelsheight was excluded from device")
def test_evidencekey_filter_contains_user_agent(self):
"""!
diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_missing_property_raises.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_missing_property_raises.py
new file mode 100644
index 00000000..2fe00eaa
--- /dev/null
+++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_missing_property_raises.py
@@ -0,0 +1,56 @@
+# *********************************************************************
+# This Original Work is copyright of 51 Degrees Mobile Experts Limited.
+# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House,
+# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU.
+#
+# This Original Work is licensed under the European Union Public Licence
+# (EUPL) v.1.2 and is subject to its terms as set out below.
+#
+# If a copy of the EUPL was not distributed with this file, You can obtain
+# one at https://opensource.org/licenses/EUPL-1.2.
+#
+# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be
+# amended by the European Commission) shall be deemed incompatible for
+# the purposes of the Work and the provisions of the compatibility
+# clause in Article 5 of the EUPL shall not apply.
+#
+# If using the Work as, or as part of, a network application, by
+# including the attribution notice(s) required under Article 5 of the EUPL
+# in the end user terms of the application under an appropriate heading,
+# such notice(s) shall fulfill the requirements of that article.
+# *********************************************************************
+
+"""A property the engine lists, in a data file tier that holds it, raises
+when the data has no value for it, as every missing property does."""
+
+import unittest
+from types import SimpleNamespace
+
+from fiftyone_devicedetection_onpremise.swig_data import \
+ OnPremiseMissingPropertyService
+
+
+class StandInEngine:
+ """An engine whose data file is Lite, listing one property that the
+ Lite file holds."""
+
+ datakey = "device"
+ properties = {"ismobile": {"datafiles": ["Lite", "Enterprise"]}}
+ engine = SimpleNamespace(getProduct=lambda: "Lite")
+
+
+class MissingPropertyRaisesTests(unittest.TestCase):
+
+ def test_a_listed_property_of_this_tier_raises(self):
+ with self.assertRaises(Exception) as raised:
+ OnPremiseMissingPropertyService().check(
+ "ismobile", StandInEngine())
+
+ self.assertEqual(
+ "Property 'ismobile' not found in data for element 'device'. "
+ "This is because the property has been excluded when "
+ "configuring the engine.", str(raised.exception))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py
index 2070a22b..85fa7094 100644
--- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py
+++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py
@@ -27,8 +27,7 @@
from fiftyone_pipeline_engines.aspectdata import AspectData
from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue
-from fiftyone_pipeline_engines.missingproperty_service import \
- OnPremiseMissingPropertyService
+from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService
from .constants import RESULTS_CLOSED
@@ -43,6 +42,24 @@
COORDINATE_PROPERTIES = ("latitude", "longitude")
+class OnPremiseMissingPropertyService(MissingPropertyService):
+
+ def check(self, key, flow_element):
+
+ if key in flow_element.properties:
+ datafiles = flow_element.properties[key]["datafiles"]
+ if(not flow_element.engine.getProduct() in datafiles):
+ raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. This is because your datafile does not contain the property. The property is available in' + str(datafiles))
+ # The data file holds the property, so it is missing because
+ # it is not available. A missing property always raises.
+ raise Exception(
+ "Property '" + key + "' not found in data for element '" +
+ flow_element.datakey + "'. This is because the property "
+ "has been excluded when configuring the engine.")
+ else:
+ raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. Please check that the element and property names are correct.')
+
+
class SwigData(AspectData):
"""!
diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_missing_property_raises.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_missing_property_raises.py
new file mode 100644
index 00000000..f8e282e7
--- /dev/null
+++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_missing_property_raises.py
@@ -0,0 +1,56 @@
+# *********************************************************************
+# This Original Work is copyright of 51 Degrees Mobile Experts Limited.
+# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House,
+# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU.
+#
+# This Original Work is licensed under the European Union Public Licence
+# (EUPL) v.1.2 and is subject to its terms as set out below.
+#
+# If a copy of the EUPL was not distributed with this file, You can obtain
+# one at https://opensource.org/licenses/EUPL-1.2.
+#
+# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be
+# amended by the European Commission) shall be deemed incompatible for
+# the purposes of the Work and the provisions of the compatibility
+# clause in Article 5 of the EUPL shall not apply.
+#
+# If using the Work as, or as part of, a network application, by
+# including the attribution notice(s) required under Article 5 of the EUPL
+# in the end user terms of the application under an appropriate heading,
+# such notice(s) shall fulfill the requirements of that article.
+# *********************************************************************
+
+"""A property the engine lists, in a data file tier that holds it, raises
+when the data has no value for it, as every missing property does."""
+
+import unittest
+from types import SimpleNamespace
+
+from fiftyone_ipintelligence_onpremise.swig_data import \
+ OnPremiseMissingPropertyService
+
+
+class StandInEngine:
+ """An engine whose data file is Lite, listing one property that the
+ Lite file holds."""
+
+ datakey = "ip"
+ properties = {"registeredname": {"datafiles": ["Lite", "Enterprise"]}}
+ engine = SimpleNamespace(getProduct=lambda: "Lite")
+
+
+class MissingPropertyRaisesTests(unittest.TestCase):
+
+ def test_a_listed_property_of_this_tier_raises(self):
+ with self.assertRaises(Exception) as raised:
+ OnPremiseMissingPropertyService().check(
+ "registeredname", StandInEngine())
+
+ self.assertEqual(
+ "Property 'registeredname' not found in data for element 'ip'. "
+ "This is because the property has been excluded when "
+ "configuring the engine.", str(raised.exception))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py
index e45f67fd..4f4f8c88 100644
--- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py
+++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py
@@ -22,48 +22,27 @@
from fiftyone_pipeline_engines.aspectdata_dictionary import AspectDataDictionary
-from fiftyone_pipeline_engines.missingproperty_service import \
- MissingPropertyReason, MissingPropertyService
-
-
-class CloudMissingPropertyService(MissingPropertyService):
-
- """!
- The missing property service of a Cloud Aspect Engine. A property its
- metadata does not list is one the key in use does not include, or,
- while the engine's property list is not known, one whose request for
- that list failed.
- """
-
- def engine_reason(self, flow_element):
-
- """!
- The reason for a property the metadata does not list.
-
- @type flow_element: CloudEngine
- @param flow_element: the engine
- @rtype: tuple
- @return: the reason, and the property names the key includes
- """
-
- properties_known = getattr(flow_element, "properties_known", None)
- if properties_known is not None and not properties_known():
- # The Cloud Request Engine asks for the property list when it is
- # built, so a list that is still not known is one whose request
- # failed.
- return MissingPropertyReason.CLOUD_REQUEST_FAILED, ()
- properties = self.properties(flow_element)
- if not properties:
- return (
- MissingPropertyReason.PRODUCT_NOT_ACCESSIBLE_WITH_RESOURCE_KEY,
- ())
- return (
- MissingPropertyReason.PROPERTY_NOT_ACCESSIBLE_WITH_RESOURCE_KEY,
- [meta.get("name", name) for name, meta in properties.items()])
-
-
-# The name this service had before, kept for code that imports it.
-OnPremiseMissingPropertyService = CloudMissingPropertyService
+from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService
+
+class OnPremiseMissingPropertyService(MissingPropertyService):
+
+ def check(self, key, flow_element):
+
+ # The properties listed here are the ones the key carries, which is
+ # not the same as the ones one answer holds: a request made with a
+ # licence key carries only the properties it asked for, so a
+ # property can be listed and still be absent. The message names
+ # both causes, so the reader checks the right one.
+ raise Exception(
+ "Property " + key + " not found in data for element "
+ + flow_element.datakey + ". Either the key in use does not "
+ "include access to this property, or the request was made "
+ "with a licence key and the property was not among the "
+ "requested_properties. Properties the key carries under "
+ + flow_element.datakey + " are "
+ + ", ".join(list(flow_element.get_properties().keys()))
+ + ". For more details on resource keys, see our explainer: "
+ "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-clouddata.py&utm_term=property-not-found")
class CloudData(AspectDataDictionary):
@@ -83,4 +62,4 @@ def __init__(self, flow_element, contents):
super(CloudData, self).__init__(flow_element, contents)
- self.missing_property_service = CloudMissingPropertyService()
+ self.missing_property_service = OnPremiseMissingPropertyService()
diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py
index 1352adde..248ec92e 100644
--- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py
+++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py
@@ -26,12 +26,14 @@
from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue
from fiftyone_pipeline_core.jsonbundler import WKT_TYPES
-from .clouddata import CloudData, CloudMissingPropertyService
+from .clouddata import CloudData
from .constants import Constants
import json
import threading
+from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService
+
# Why a pipeline with a second CloudRequestEngine is refused.
_ANOTHER_REQUEST_ENGINE = (
"This cloud engine is already in a pipeline with another Cloud Request "
@@ -39,9 +41,11 @@
"pipeline with a different one is refused. Build another cloud engine "
"for this pipeline, or use the same Cloud Request Engine in both.")
-# The name the cloud missing property service had here, kept for code that
-# imports it.
-OnPremiseMissingPropertyService = CloudMissingPropertyService
+class OnPremiseMissingPropertyService(MissingPropertyService):
+
+ def check(self, key, flow_element):
+
+ raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. Please check that the element and property names are correct.')
class CloudEngine(Engine):
@@ -128,20 +132,6 @@ def get_properties(self):
self._load_properties(self._log)
return super(CloudEngine, self).get_properties()
- def properties_known(self):
-
- """!
- Whether this engine has its properties from the CloudRequestEngine,
- which it has once that engine's discovery has completed. The
- properties are looked for first, as get_properties does.
-
- @rtype: bool
- @return: True once the properties are known
- """
-
- self._load_properties(self._log)
- return self._properties_final
-
def _load_properties(self, log):
"""!
diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_property.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_property.py
deleted file mode 100644
index 92117abc..00000000
--- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_property.py
+++ /dev/null
@@ -1,133 +0,0 @@
-# *********************************************************************
-# This Original Work is copyright of 51 Degrees Mobile Experts Limited.
-# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House,
-# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU.
-#
-# This Original Work is licensed under the European Union Public Licence
-# (EUPL) v.1.2 and is subject to its terms as set out below.
-#
-# If a copy of the EUPL was not distributed with this file, You can obtain
-# one at https://opensource.org/licenses/EUPL-1.2.
-#
-# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be
-# amended by the European Commission) shall be deemed incompatible for
-# the purposes of the Work and the provisions of the compatibility
-# clause in Article 5 of the EUPL shall not apply.
-#
-# If using the Work as, or as part of, a network application, by
-# including the attribution notice(s) required under Article 5 of the EUPL
-# in the end user terms of the application under an appropriate heading,
-# such notice(s) shall fulfill the requirements of that article.
-# *********************************************************************
-
-"""
-The reason a Cloud Aspect Engine gives for a missing property, decided in
-the order the properties specification sets out, with the message of the
-missing properties table for that reason.
-"""
-
-import json
-import unittest
-
-from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine
-from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary
-from fiftyone_pipeline_core.flowelement import FlowElement
-from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder
-
-# The configurator the Resource Key rows link to, with this repository's
-# campaign parameters.
-CONFIGURATOR = "https://configure.51degrees.com/?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-configurator"
-
-
-class FakeCloudRequestEngine(FlowElement):
- """Stands in for the Cloud Request Engine of a key that carries only the
- device product, with two of its properties."""
-
- def __init__(self):
- super(FakeCloudRequestEngine, self).__init__()
- self.datakey = "cloud"
- self.flow_element_properties = {"device": {
- "ismobile": {"name": "IsMobile", "type": "Boolean"},
- "hardwarevendor": {"name": "HardwareVendor", "type": "String"}}}
-
- def process_internal(self, flowdata):
- flowdata.set_element_data(ElementDataDictionary(
- self, {"cloud": json.dumps({"device": {"ismobile": True}})}))
-
-
-class UndiscoveredCloudRequestEngine(FakeCloudRequestEngine):
- """Stands in for a Cloud Request Engine whose request for the property
- list failed when it was built, so its properties are not known."""
-
- def known_flow_element_properties(self):
- return None
-
-
-def build(request_engine, datakey, restricted=None):
- engine = CloudEngine()
- engine.datakey = datakey
- if restricted is not None:
- engine.set_restricted_properties(restricted)
- pipeline = PipelineBuilder({
- "add_javascript_builder": False,
- "use_setheader_properties": False}).add(
- request_engine).add(engine).build()
- flowdata = pipeline.create_flowdata()
- flowdata.process()
- return pipeline, flowdata
-
-
-class CloudReasonTests(unittest.TestCase):
-
- def message(self, request_engine, datakey, name, restricted=None):
- pipeline, flowdata = build(request_engine, datakey, restricted)
- self.addCleanup(pipeline.close)
- with self.assertRaises(Exception) as raised:
- flowdata.get(datakey).get(name)
- return str(raised.exception)
-
- def test_a_product_the_key_does_not_include(self):
- self.assertEqual(
- "Property 'country' not found in data for element 'location'. "
- "This is because your resource key does not include access to "
- "any properties under 'location'. Check or create a resource key "
- "with the properties you need in the configurator at "
- + CONFIGURATOR + ".",
- self.message(FakeCloudRequestEngine(), "location", "country"))
-
- def test_a_property_the_key_does_not_include(self):
- self.assertEqual(
- "Property 'browsername' not found in data for element 'device'. "
- "This is because your resource key does not include access to "
- "this property. The properties included for your key under "
- "'device' are IsMobile,HardwareVendor. Check or change your key "
- "in the configurator at " + CONFIGURATOR + ".",
- self.message(FakeCloudRequestEngine(), "device", "browsername"))
-
- def test_a_listed_property_not_in_the_answer_was_excluded(self):
- self.assertEqual(
- "Property 'HardwareVendor' not found in data for element "
- "'device'. This is because the property has been excluded when "
- "configuring the engine.",
- self.message(
- FakeCloudRequestEngine(), "device", "HardwareVendor"))
-
- def test_a_property_left_out_of_the_restricted_list_was_excluded(self):
- self.assertIn(
- "This is because the property has been excluded when "
- "configuring the engine.",
- self.message(
- FakeCloudRequestEngine(), "device", "hardwarevendor",
- restricted=["IsMobile"]))
-
- def test_a_failed_request_for_the_property_list(self):
- self.assertEqual(
- "Property 'browsername' not found in data for element 'device'. "
- "This is because an upstream cloud request failed, so no data "
- "was available to populate the property.",
- self.message(
- UndiscoveredCloudRequestEngine(), "device", "browsername"))
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py
index 1f42abc6..72b5a11c 100644
--- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py
+++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py
@@ -595,12 +595,13 @@ def test_no_report_when_every_property_came_back(self):
str(warning.message) for warning in caught
if "not covered" in str(warning.message)])
- def test_missing_property_message_names_the_properties_carried(self):
+ def test_missing_property_message_names_both_causes(self):
"""
- With a licence key, a property the key does not carry gives the
- missing properties table's message for it, which lists the
- properties the key carries under the element it looked in rather
- than assuming device.
+ With a licence key the accessible properties list everything the
+ licence entitles, so a property can be listed as carried and still
+ be absent from an answer because it was not requested. The message
+ for a property that is not in the data says so, and names the
+ element it looked in rather than assuming device.
"""
engine = CloudRequestEngine({
@@ -620,8 +621,7 @@ def test_missing_property_message_names_the_properties_carried(self):
data.device.get("other")
message = str(context.exception)
- self.assertIn(
- "Property 'other' not found in data for element 'device'. This "
- "is because your resource key does not include access to this "
- "property. The properties included for your key under 'device' "
- "are value.", message)
+ self.assertIn("Property other not found", message)
+ self.assertIn("requested_properties", message)
+ self.assertIn("under device are value", message)
+ self.assertNotIn("your resource key", message)
diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py
index 28faf87e..fa02ff16 100644
--- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py
+++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py
@@ -61,17 +61,16 @@ def get(self, key):
@rtype mixed
@return The value stored under the chosen key
@raise Exception: where the property was left out of the engine's
- restricted properties, or the data has no value for it, with the
- message the missing property service gives for the reason
+ restricted properties, saying so, or where the data has no value
+ for it, with the message the missing property service gives
"""
if hasattr(self.flow_element, "restricted_property_keys"):
if key.lower() not in self.flow_element.restricted_property_keys:
- # The service gives the reason, as the property was
- # excluded or is in a data tier the data file lacks.
- return self.missing_property_service.check(
- key, self.flow_element)
+ raise Exception(
+ "Property " + key + " was excluded from " +
+ self.flow_element.datakey)
try:
if self.get_internal(key) == None:
diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/missingproperty_service.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/missingproperty_service.py
index 35c4a7c6..c99faa00 100644
--- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/missingproperty_service.py
+++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/missingproperty_service.py
@@ -20,104 +20,13 @@
# such notice(s) shall fulfill the requirements of that article.
# *********************************************************************
-from enum import Enum
-
-
-class MissingPropertyReason(Enum):
- """!
- Why a property an engine defines is missing from its data, named as the
- .NET packages name each reason.
- """
-
- DATA_FILE_UPGRADE_REQUIRED = "DataFileUpgradeRequired"
- PROPERTY_EXCLUDED_FROM_ENGINE_CONFIGURATION = \
- "PropertyExcludedFromEngineConfiguration"
- PRODUCT_NOT_ACCESSIBLE_WITH_RESOURCE_KEY = \
- "ProductNotAccessibleWithResourceKey"
- PROPERTY_NOT_ACCESSIBLE_WITH_RESOURCE_KEY = \
- "PropertyNotAccessibleWithResourceKey"
- CLOUD_REQUEST_FAILED = "CloudRequestFailed"
- UNKNOWN = "Unknown"
-
-
-# The pages the messages link to, with this repository's campaign
-# parameters.
-PRICING_URL = "https://51degrees.com/pricing?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-pricing"
-CONFIGURATOR_URL = "https://configure.51degrees.com/?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-configurator"
-
-# Lists in a message are joined with a comma and no space.
-LIST_SEPARATOR = ","
-
-
-def missing_property_message(
- reason, name, element, tiers=(), engine_type="", properties=()):
- """!
- The message of the missing properties table for a reason.
-
- @type reason: MissingPropertyReason
- @param reason: why the property is missing
- @type name: str
- @param name: the property, as the caller named it
- @type element: str
- @param element: the data key of the element asked
- @type tiers: list
- @param tiers: the data tiers that hold the property, for
- DATA_FILE_UPGRADE_REQUIRED, where an empty list gives the row for
- tiers that are not known
- @type engine_type: str
- @param engine_type: the engine's type name, for
- DATA_FILE_UPGRADE_REQUIRED
- @type properties: list
- @param properties: the property names the key includes, for
- PROPERTY_NOT_ACCESSIBLE_WITH_RESOURCE_KEY
- @rtype: str
- @return: the message
- """
-
- text = ("Property '" + name + "' not found in data for element '" +
- element + "'. ")
- if reason is MissingPropertyReason.DATA_FILE_UPGRADE_REQUIRED:
- text += ("This is because your license or data file does not "
- "include this property. ")
- if tiers:
- return text + (
- "It is available with the " + LIST_SEPARATOR.join(tiers) +
- " license/data for the " + engine_type +
- ". To upgrade, see " + PRICING_URL + ".")
- return text + (
- "To see which licenses include it, see " + PRICING_URL + ".")
- if reason is MissingPropertyReason.PROPERTY_EXCLUDED_FROM_ENGINE_CONFIGURATION:
- return text + ("This is because the property has been excluded "
- "when configuring the engine.")
- if reason is MissingPropertyReason.PRODUCT_NOT_ACCESSIBLE_WITH_RESOURCE_KEY:
- return text + (
- "This is because your resource key does not include access to "
- "any properties under '" + element + "'. Check or create a "
- "resource key with the properties you need in the configurator "
- "at " + CONFIGURATOR_URL + ".")
- if reason is MissingPropertyReason.PROPERTY_NOT_ACCESSIBLE_WITH_RESOURCE_KEY:
- return text + (
- "This is because your resource key does not include access to "
- "this property. The properties included for your key under '" +
- element + "' are " + LIST_SEPARATOR.join(properties) +
- ". Check or change your key in the configurator at " +
- CONFIGURATOR_URL + ".")
- if reason is MissingPropertyReason.CLOUD_REQUEST_FAILED:
- return text + (
- "This is because an upstream cloud request failed, so no data "
- "was available to populate the property.")
- return text + ("The reason for this is unknown. Please check that the "
- "aspect and property name are correct.")
-
class MissingPropertyService():
"""!
- Raises the error for a property an engine's data does not hold, giving
- the reason in the order the properties specification sets out, with
- the missing properties table's message for it. Property names are
- compared ignoring case. Engines of a kind override needs_upgrade and
- engine_reason to add their own steps.
+ A missing property service runs when a property is not available in the
+ aspectData. It can be extended to provide a specific message for why the property
+ is not available
"""
def check(self, key, flow_element):
@@ -129,121 +38,7 @@ def check(self, key, flow_element):
@param key: the property asked for
@type flow_element: FlowElement
@param flow_element: the element whose data was asked
- @raise Exception: always, with the message for the reason the
- property is missing
- """
-
- raise Exception(self.message(key, flow_element))
-
- def message(self, key, flow_element):
-
- """!
- The message for the reason a property is missing.
-
- @type key: string
- @param key: the property asked for
- @type flow_element: FlowElement
- @param flow_element: the element whose data was asked
- @rtype: str
- @return: the message
- """
-
- name = key.lower()
- listed = None
- for property_name, meta in self.properties(flow_element).items():
- if property_name.lower() == name:
- listed = meta
- break
- restricted = getattr(flow_element, "restricted_property_keys", None)
-
- # A property the metadata lists, or one the restricted list leaves
- # out, is in a data tier the data file does not hold, or was
- # excluded.
- if listed is not None or (
- restricted is not None and name not in restricted):
- tiers = list((listed or {}).get("datafiles") or ())
- if listed is not None and self.needs_upgrade(tiers, flow_element):
- return missing_property_message(
- MissingPropertyReason.DATA_FILE_UPGRADE_REQUIRED, key,
- flow_element.datakey, tiers=tiers,
- engine_type=type(flow_element).__name__)
- return missing_property_message(
- MissingPropertyReason.PROPERTY_EXCLUDED_FROM_ENGINE_CONFIGURATION,
- key, flow_element.datakey)
-
- reason, properties = self.engine_reason(flow_element)
- return missing_property_message(
- reason, key, flow_element.datakey, properties=properties)
-
- def properties(self, flow_element):
-
- """!
- The engine's property metadata, by name.
-
- @type flow_element: FlowElement
- @param flow_element: the engine
- @rtype: dict
- @return: the metadata of each property, by name
- """
-
- get_properties = getattr(flow_element, "get_properties", None)
- if get_properties is not None:
- return get_properties() or {}
- return getattr(flow_element, "properties", None) or {}
-
- def needs_upgrade(self, tiers, flow_element):
-
- """!
- Whether a property the metadata lists is missing because the
- engine's data file is not of one of the property's tiers. An engine
- with no data file never needs an upgrade.
-
- @type tiers: list
- @param tiers: the data tiers that hold the property
- @type flow_element: FlowElement
- @param flow_element: the engine
- @rtype: bool
- @return: True where an upgrade is needed
- """
-
- return False
-
- def engine_reason(self, flow_element):
-
- """!
- The reason for a property the metadata does not list, which a kind
- of engine with a step of its own overrides.
-
- @type flow_element: FlowElement
- @param flow_element: the engine
- @rtype: tuple
- @return: the reason, and the property names for a message that
- lists them
- """
-
- return MissingPropertyReason.UNKNOWN, ()
-
-
-class OnPremiseMissingPropertyService(MissingPropertyService):
-
- """!
- The missing property service of an on-premise engine, whose data file
- tier decides whether a property the metadata lists needs an upgrade.
- The engine's native engine gives the tier.
- """
-
- def needs_upgrade(self, tiers, flow_element):
-
- """!
- Whether the tier of the data file the engine uses is not among a
- property's tiers.
-
- @type tiers: list
- @param tiers: the data tiers that hold the property
- @type flow_element: FlowElement
- @param flow_element: the engine
- @rtype: bool
- @return: True where an upgrade is needed
+ @raise Exception: always, saying why the property is missing
"""
- return flow_element.engine.getProduct() not in tiers
+ raise Exception("Property " + key + " not found in " + flow_element.datakey)
diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_engines.py b/pipeline/fiftyone_pipeline_engines/tests/test_engines.py
index 86553b9c..3b30e850 100644
--- a/pipeline/fiftyone_pipeline_engines/tests/test_engines.py
+++ b/pipeline/fiftyone_pipeline_engines/tests/test_engines.py
@@ -140,10 +140,7 @@ def testRestrictedProperty(self):
except Exception as e:
result = str(e)
- self.assertEqual(
- "Property 'integer' not found in data for element 'example'. "
- "This is because the property has been excluded when configuring "
- "the engine.", result)
+ self.assertTrue(result == "Property integer was excluded from example")
# Restricted property names are compared ignoring case
def testRestrictedPropertyIgnoresCase(self):
@@ -167,9 +164,7 @@ def testRestrictedPropertyIgnoresCase(self):
with self.assertRaises(Exception) as refused:
flowdata.example.get(key)
self.assertEqual(
- "Property '" + key + "' not found in data for element "
- "'example'. This is because the property has been excluded "
- "when configuring the engine.",
+ "Property " + key + " was excluded from example",
str(refused.exception))
# Test restricting properties
@@ -188,10 +183,7 @@ def testMissingPropertyService(self):
except Exception as e:
result = str(e)
- self.assertEqual(
- "Property 'missing' not found in data for element 'example'. "
- "The reason for this is unknown. Please check that the aspect and "
- "property name are correct.", result)
+ self.assertTrue(result == "Property missing not found in example")
# Test cache
diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_missing_property_reasons.py b/pipeline/fiftyone_pipeline_engines/tests/test_missing_property_reasons.py
deleted file mode 100644
index c9944ed4..00000000
--- a/pipeline/fiftyone_pipeline_engines/tests/test_missing_property_reasons.py
+++ /dev/null
@@ -1,121 +0,0 @@
-# *********************************************************************
-# This Original Work is copyright of 51 Degrees Mobile Experts Limited.
-# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House,
-# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU.
-#
-# This Original Work is licensed under the European Union Public Licence
-# (EUPL) v.1.2 and is subject to its terms as set out below.
-#
-# If a copy of the EUPL was not distributed with this file, You can obtain
-# one at https://opensource.org/licenses/EUPL-1.2.
-#
-# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be
-# amended by the European Commission) shall be deemed incompatible for
-# the purposes of the Work and the provisions of the compatibility
-# clause in Article 5 of the EUPL shall not apply.
-#
-# If using the Work as, or as part of, a network application, by
-# including the attribution notice(s) required under Article 5 of the EUPL
-# in the end user terms of the application under an appropriate heading,
-# such notice(s) shall fulfill the requirements of that article.
-# *********************************************************************
-
-"""
-The reason an on-premise engine gives for a missing property, decided in
-the order the properties specification sets out, with the message of the
-missing properties table for that reason.
-"""
-
-import unittest
-
-from fiftyone_pipeline_engines.missingproperty_service import \
- OnPremiseMissingPropertyService
-
-# The pricing page the data file rows link to, with this repository's
-# campaign parameters.
-PRICING = "https://51degrees.com/pricing?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-pricing"
-
-
-class StandInNativeEngine:
- """Stands in for the native engine, giving the data file's tier."""
-
- def __init__(self, tier):
- self.tier = tier
-
- def getProduct(self):
- return self.tier
-
-
-class StandInEngine:
- """Stands in for an on-premise engine, with its property metadata keyed
- by lower-case name as the engines keep it."""
-
- def __init__(self, tier, properties):
- self.datakey = "device"
- self.engine = StandInNativeEngine(tier)
- self.properties = properties
-
- def get_properties(self):
- return self.properties
-
-
-def message(name, engine):
- with unittest.TestCase().assertRaises(Exception) as raised:
- OnPremiseMissingPropertyService().check(name, engine)
- return str(raised.exception)
-
-
-class OnPremiseReasonTests(unittest.TestCase):
-
- def test_a_property_of_other_tiers_needs_an_upgrade(self):
- engine = StandInEngine("Lite", {
- "screenpixelsheight": {"datafiles": ["Premium", "Enterprise"]}})
- self.assertEqual(
- "Property 'ScreenPixelsHeight' not found in data for element "
- "'device'. This is because your license or data file does not "
- "include this property. It is available with the "
- "Premium,Enterprise license/data for the StandInEngine. To "
- "upgrade, see " + PRICING + ".",
- message("ScreenPixelsHeight", engine))
-
- def test_a_property_with_no_tiers_known_needs_an_upgrade(self):
- engine = StandInEngine("Lite", {"screenpixelsheight": {
- "datafiles": []}})
- self.assertEqual(
- "Property 'screenpixelsheight' not found in data for element "
- "'device'. This is because your license or data file does not "
- "include this property. To see which licenses include it, see "
- + PRICING + ".",
- message("screenpixelsheight", engine))
-
- def test_a_property_of_this_tier_was_excluded(self):
- engine = StandInEngine("Enterprise", {
- "screenpixelsheight": {"datafiles": ["Premium", "Enterprise"]}})
- self.assertEqual(
- "Property 'screenpixelsheight' not found in data for element "
- "'device'. This is because the property has been excluded when "
- "configuring the engine.",
- message("screenpixelsheight", engine))
-
- def test_a_property_the_metadata_does_not_list_is_unknown(self):
- engine = StandInEngine("Enterprise", {
- "screenpixelsheight": {"datafiles": ["Enterprise"]}})
- self.assertEqual(
- "Property 'notthere' not found in data for element 'device'. "
- "The reason for this is unknown. Please check that the aspect "
- "and property name are correct.",
- message("notthere", engine))
-
- def test_names_are_compared_ignoring_case(self):
- engine = StandInEngine("Lite", {
- "screenpixelsheight": {"datafiles": ["Enterprise"]}})
- for name in ("screenpixelsheight", "SCREENPIXELSHEIGHT",
- "ScreenPixelsHeight"):
- with self.subTest(name=name):
- self.assertIn(
- "It is available with the Enterprise license/data",
- message(name, engine))
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py
index 88cb2f61..74ca6f9c 100644
--- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py
+++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py
@@ -144,11 +144,10 @@ def __init__(
needs cookie to be set, and works the same whether or not
share_all is True.
@type query_whitelist: list or str
- @param query_whitelist: The query string and HTTP form parameters
- to share, as well as those prefixed with '51D_', when
- share_all_query_string_parameters is False. A string of names
- separated by commas, which is how a configuration file gives a
- list, is read as those names. None holds no names.
+ @param query_whitelist: Deprecated, and has no effect. A query
+ string or HTTP form parameter is shared only when its name starts
+ with '51D_' or the Pipeline uses it as evidence, whatever the
+ settings.
@type header_blacklist: list or str
@param header_blacklist: By default, all HTTP headers
except Cookie are shared.
@@ -160,13 +159,13 @@ def __init__(
@type share_all: bool
@param share_all: If True, all evidence is shared apart from the
email address, the salt used for hashed email identifiers, the
- request's query string and path, and every cookie whose name does
- not start with '51D_'. False by default.
+ request's query string and path, every cookie whose name does not
+ start with '51D_', and every query string or HTTP form parameter
+ whose name does not start with '51D_' and that the Pipeline does
+ not use as evidence. False by default.
@type share_all_query_string_parameters: bool
- @param share_all_query_string_parameters: If True, every query
- string and HTTP form parameter is shared. If False, only those
- prefixed with '51D_' and those in query_whitelist are. True by
- default.
+ @param share_all_query_string_parameters: Deprecated, and has no
+ effect, as query_whitelist.
@type max_queue_size: int
@param max_queue_size: The most entries held while waiting to be
sent. When the queue is full, new entries are dropped. The default
@@ -741,7 +740,7 @@ def process_internal(self, flow_data):
# A hash of the values that would be shared, as the element
# accepts no evidence of its own.
cachekey = self.tracker_evidence_key_filter.filter_evidence(
- self.found_evidence(flow_data))
+ self.found_evidence(flow_data), flow_data.pipeline)
# Evidence keys ignore case, so a repeat is found whatever the
# letters of its keys.
cachekey = json.dumps(
@@ -784,7 +783,8 @@ def get_data_from_evidence(self, flow_data):
f"The value '{value}' could not be parsed to an integer.")
else:
# Check if we can send this piece of evidence
- if self.data_evidence_key_filter.filter_evidence_key(key):
+ if self.data_evidence_key_filter.filter_evidence_key(
+ key, flow_data.pipeline):
data.try_add(key, value)
return data
diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py
index 9438ff5a..e37fc724 100644
--- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py
+++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py
@@ -21,6 +21,7 @@
# *********************************************************************
from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter
+from fiftyone_pipeline_core.javascriptbuilder import JavascriptBuilderElement
# Cookies, session values and query string parameters whose names start with
# this are the ones 51Degrees sets, and are shared.
@@ -37,6 +38,30 @@
"id.email", "id.salt", "request-query", "request-path", "header.cookie")
+# The query string parameters the JavaScript Builder's specification lists as
+# its accepted Evidence. Its filter also passes every other parameter, only so
+# that it can send them back to the callback.
+JAVASCRIPT_BUILDER_PARAMETERS = (
+ "query.fod-js-object-name", "query.fod-js-enable-cookies")
+
+
+def _used_by(pipeline, key):
+ """!
+ Whether a FlowElement of the Pipeline accepts the key as evidence. The
+ JavaScript Builder is asked only about the parameters its specification
+ says it accepts.
+ """
+ if pipeline is None:
+ return False
+ for element in pipeline.flow_elements:
+ if isinstance(element, JavascriptBuilderElement):
+ if key.lower() in JAVASCRIPT_BUILDER_PARAMETERS:
+ return True
+ elif element.get_evidence_key_filter().filter_evidence_key(key):
+ return True
+ return False
+
+
def _split_names(names):
"""!
A configuration file gives a list of names as one comma-separated
@@ -53,10 +78,10 @@ class ShareUsageEvidenceKeyFilter(BasicListEvidenceKeyFilter):
"""
The ShareUsageEvidenceKeyFilter decides which evidence the 51Degrees
ShareUsage service is sent. Cookies are shared only when their names
- start with 51D_, whatever the other settings. Headers are shared unless
- they are blocked, session values only when their names start with 51D_,
- and query string and form parameters when all of them are shared or
- when their names start with 51D_ or are on the list to share. Any other
+ start with 51D_, and query string and form parameters only when their
+ names start with 51D_ or the Pipeline uses them as evidence, whatever
+ the other settings. Headers are shared unless they are blocked, and
+ session values only when their names start with 51D_. Any other
evidence is shared.
"""
@@ -73,8 +98,9 @@ def __init__(
@type cookie: string
@param cookie: the name of the cookie that identifies the session
@type query_whitelist: list or str
- @param query_whitelist: query string and form parameters to share as
- well as those starting 51D_. None holds no names. A string is read as names separated by commas.
+ @param query_whitelist: deprecated, and has no effect. A query
+ string or form parameter is shared only when its name starts with
+ 51D_ or the Pipeline uses it as evidence.
@type header_blacklist: list or str
@param header_blacklist: headers not to share. A string is read as
names separated by commas.
@@ -84,14 +110,16 @@ def __init__(
@param share_all: whether all evidence is shared, apart from the
evidence that is never shared and cookies not starting 51D_
@type share_all_query_string_parameters: bool
- @param share_all_query_string_parameters: whether every query
- string and form parameter is shared, whatever query_whitelist holds
+ @param share_all_query_string_parameters: deprecated, and has no
+ effect, as query_whitelist.
"""
self.cookie = cookie.lower() if cookie else None
query_whitelist = _split_names(query_whitelist)
header_blacklist = _split_names(header_blacklist)
+ # Kept so that code reading them still works. Neither decides what
+ # is shared.
self.query_whitelist = \
[parameter.lower() for parameter in query_whitelist or ()]
@@ -105,18 +133,40 @@ def __init__(
self.share_all_query_string_parameters = \
share_all_query_string_parameters
- def filter_evidence_key(self, key):
+ def filter_evidence(self, evidence_key_object, pipeline=None):
+
+ """!
+ The evidence to share, from a dictionary of evidence.
+
+ @type evidence_key_object: dict
+ @param evidence_key_object: evidence keys and values
+ @type pipeline: Pipeline
+ @param pipeline: the Pipeline the evidence came to, whose evidence
+ decides which query string and form parameters are shared
+ @rtype: dict
+ @return: the evidence to share
+ """
+
+ return {key: value for key, value in evidence_key_object.items()
+ if self.filter_evidence_key(key, pipeline)}
+
+ def filter_evidence_key(self, key, pipeline=None):
"""!
Check if a specific key should be filtered.
@type key: string
@param key: to check in the filter
+ @type pipeline: Pipeline
+ @param pipeline: the Pipeline the evidence came to. Without one, a
+ query string or form parameter is shared only when its name starts
+ with 51D_.
@rtype: bool
@return: Is this key in the filter's keys list?
"""
+ original = key
key = key.lower()
if any(key == never_shared or key.endswith("." + never_shared)
@@ -133,6 +183,13 @@ def filter_evidence_key(self, key):
return field.startswith(FIFTYONE_PREFIX) or \
(self.include_session and field == self.cookie)
+ # A query string or form parameter is shared only when 51Degrees set
+ # it or the Pipeline uses it as evidence, even when all evidence is,
+ # so that a value such as a password is never sent.
+ if separator and prefix == "query":
+ return field.startswith(FIFTYONE_PREFIX) or \
+ _used_by(pipeline, original)
+
if self.share_all:
return True
@@ -146,12 +203,6 @@ def filter_evidence_key(self, key):
# Only session values that start with the 51Degrees
# cookie prefix
return field.startswith(FIFTYONE_PREFIX)
- elif prefix == "query":
- # Share every parameter only when asked to. Otherwise only
- # those starting 51d_ or in the list.
- return self.share_all_query_string_parameters or \
- field.startswith(FIFTYONE_PREFIX) or \
- field in self.query_whitelist
else:
# Add anything that is not a cookie, header, session
# variable or query parameter
@@ -164,10 +215,10 @@ class ShareUsageTrackerEvidenceKeyFilter(ShareUsageEvidenceKeyFilter):
out, because they make every request different.
"""
- def filter_evidence_key(self, key):
+ def filter_evidence_key(self, key, pipeline=None):
if key.lower() in ("query.session-id", "query.sequence"):
return False
return super(ShareUsageTrackerEvidenceKeyFilter, self)\
- .filter_evidence_key(key)
+ .filter_evidence_key(key, pipeline)
diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py
index 0256ca15..eb3a0fdc 100644
--- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py
+++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py
@@ -22,10 +22,31 @@
import unittest
+from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \
+ BasicListEvidenceKeyFilter
+from fiftyone_pipeline_core.flowelement import FlowElement
from fiftyone_pipeline_engines_fiftyone.share_usage_evidencekeyfilter import \
ShareUsageEvidenceKeyFilter
+class UsingElement(FlowElement):
+ """Accepts the evidence keys given."""
+
+ def __init__(self, keys):
+ super(UsingElement, self).__init__()
+ self.keys = keys
+
+ def get_evidence_key_filter(self):
+ return BasicListEvidenceKeyFilter(self.keys)
+
+
+class UsingPipeline:
+ """Stands in for a Pipeline whose FlowElement uses the keys given."""
+
+ def __init__(self, *keys):
+ self.flow_elements = [UsingElement(list(keys))]
+
+
class ShareUsageEvidenceKeyFilterTests(unittest.TestCase):
def test_default_rules(self):
@@ -53,20 +74,22 @@ def test_default_rules(self):
def test_field_is_everything_after_the_first_dot(self):
"""!
- Check that a name containing a dot is matched as a whole against
- the parameters to share.
+ Check that a parameter whose name contains a dot is matched as a
+ whole against the evidence the Pipeline uses.
"""
- keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"])
+ keys = ShareUsageEvidenceKeyFilter()
+ pipeline = UsingPipeline("query.valid.email")
- self.assertTrue(keys.filter_evidence_key("query.valid.email"))
- self.assertFalse(keys.filter_evidence_key("query.valid"))
- self.assertFalse(keys.filter_evidence_key("query.other.name"))
+ self.assertTrue(keys.filter_evidence_key("query.valid.email", pipeline))
+ self.assertFalse(keys.filter_evidence_key("query.valid", pipeline))
+ self.assertFalse(keys.filter_evidence_key("query.other.name", pipeline))
def test_lists_ignore_case(self):
"""!
- Check that the headers not to share and the parameters to share
- are matched without regard to case.
+ Check that the headers not to share are matched without regard to
+ case, and that a parameter on the deprecated list to share is not
+ shared.
"""
keys = ShareUsageEvidenceKeyFilter(
@@ -74,45 +97,55 @@ def test_lists_ignore_case(self):
header_blacklist=["X-Secret"])
self.assertFalse(keys.filter_evidence_key("header.x-secret"))
- self.assertTrue(keys.filter_evidence_key("query.campaign"))
+ self.assertFalse(keys.filter_evidence_key("query.campaign"))
self.assertTrue(keys.filter_evidence_key("header.user-agent"))
def test_lists_from_comma_separated_strings(self):
"""!
Check that a string given for either list, the form a
configuration file gives a list in, is read as names separated by
- commas, each trimmed.
+ commas, each trimmed, and that the parameters named still are not
+ shared.
"""
keys = ShareUsageEvidenceKeyFilter(
query_whitelist="Campaign, utm_source",
header_blacklist="X-Secret,X-Other")
- self.assertTrue(keys.filter_evidence_key("query.campaign"))
- self.assertTrue(keys.filter_evidence_key("query.utm_source"))
- self.assertFalse(keys.filter_evidence_key("query.c"))
+ self.assertEqual(["campaign", "utm_source"], keys.query_whitelist)
+ self.assertFalse(keys.filter_evidence_key("query.campaign"))
+ self.assertFalse(keys.filter_evidence_key("query.utm_source"))
self.assertFalse(keys.filter_evidence_key("header.x-secret"))
self.assertFalse(keys.filter_evidence_key("header.x-other"))
self.assertTrue(keys.filter_evidence_key("header.x"))
def test_no_parameter_list_shares_only_51d_parameters(self):
"""!
- Check that None in place of the parameters to share holds no
- names, as an empty list does, so only parameters starting 51D_ are
- shared unless every parameter is.
+ Check that None in place of the deprecated parameters to share, and
+ an empty list, both share only parameters starting 51D_ when the
+ Pipeline uses no others.
"""
for query_whitelist in (None, []):
- prefixed = ShareUsageEvidenceKeyFilter(
- query_whitelist=query_whitelist)
- every = ShareUsageEvidenceKeyFilter(
- query_whitelist=query_whitelist,
- share_all_query_string_parameters=True)
+ keys = ShareUsageEvidenceKeyFilter(query_whitelist=query_whitelist)
- self.assertFalse(prefixed.filter_evidence_key("query.campaign"))
- self.assertTrue(
- prefixed.filter_evidence_key("query.51d_campaign"))
- self.assertTrue(every.filter_evidence_key("query.campaign"))
+ self.assertFalse(keys.filter_evidence_key("query.campaign"))
+ self.assertTrue(keys.filter_evidence_key("query.51d_campaign"))
+
+ def test_the_deprecated_option_changes_nothing(self):
+ """!
+ Check that the deprecated share_all_query_string_parameters option
+ shares nothing more, on or off, whatever the list.
+ """
+
+ for query_whitelist in (None, [], ["campaign"]):
+ for share_every in (False, True):
+ keys = ShareUsageEvidenceKeyFilter(
+ query_whitelist=query_whitelist,
+ share_all_query_string_parameters=share_every)
+
+ self.assertFalse(keys.filter_evidence_key("query.campaign"))
+ self.assertTrue(keys.filter_evidence_key("query.51d_campaign"))
def test_session_cookie(self):
"""!
@@ -131,8 +164,9 @@ def test_session_cookie(self):
def test_share_all(self):
"""!
Check that sharing everything shares the evidence the other rules
- leave out, apart from the evidence that is never shared and the
- cookies that do not start 51D_.
+ leave out, apart from the evidence that is never shared, the
+ cookies that do not start 51D_, and the parameters that neither start
+ 51D_ nor are used by the Pipeline.
"""
keys = ShareUsageEvidenceKeyFilter(share_all=True)
@@ -141,8 +175,9 @@ def test_share_all(self):
self.assertFalse(keys.filter_evidence_key("cookie.sessionid"))
self.assertTrue(
keys.filter_evidence_key("cookie.51D_screenpixelswidth"))
- self.assertTrue(keys.filter_evidence_key("query.password"))
- self.assertTrue(keys.filter_evidence_key("query.id.usage"))
+ self.assertFalse(keys.filter_evidence_key("query.password"))
+ self.assertFalse(keys.filter_evidence_key("query.id.usage"))
+ self.assertTrue(keys.filter_evidence_key("query.51d_usage"))
self.assertTrue(keys.filter_evidence_key("server.request-method"))
self.assertFalse(keys.filter_evidence_key("query.id.email"))
self.assertFalse(keys.filter_evidence_key("header.id.email"))
@@ -153,18 +188,18 @@ def test_share_all(self):
def test_every_query_parameter_when_sharing_them_all(self):
"""!
- Check that sharing every query string parameter shares one not on
- the list, and that a filter made without it keeps to the list.
+ Check that a parameter is shared when the Pipeline uses it, and
+ another is not, whatever the deprecated options say.
"""
every = ShareUsageEvidenceKeyFilter(
- query_whitelist=["campaign"],
- share_all_query_string_parameters=True)
- listed = ShareUsageEvidenceKeyFilter(query_whitelist=["campaign"])
+ query_whitelist=["password"],
+ share_all_query_string_parameters=True,
+ share_all=True)
+ pipeline = UsingPipeline("query.campaign")
- self.assertTrue(every.filter_evidence_key("query.password"))
- self.assertFalse(listed.filter_evidence_key("query.password"))
- self.assertTrue(listed.filter_evidence_key("query.campaign"))
+ self.assertFalse(every.filter_evidence_key("query.password", pipeline))
+ self.assertTrue(every.filter_evidence_key("query.campaign", pipeline))
def test_the_session_cookie_is_kept_for_the_tracker_alone(self):
"""!
@@ -207,12 +242,13 @@ def test_never_shared_keys_match_whole_parts(self):
never shared is treated as an ordinary key.
"""
- keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"])
+ keys = ShareUsageEvidenceKeyFilter()
- self.assertTrue(keys.filter_evidence_key("query.valid.email"))
+ self.assertTrue(keys.filter_evidence_key(
+ "query.valid.email", UsingPipeline("query.valid.email")))
self.assertTrue(
ShareUsageEvidenceKeyFilter(share_all=True)
- .filter_evidence_key("query.valid.email"))
+ .filter_evidence_key("header.valid.email"))
def test_naming_a_parameter_does_not_share_the_query_string(self):
"""!
@@ -220,7 +256,9 @@ def test_naming_a_parameter_does_not_share_the_query_string(self):
string, which carries every parameter.
"""
- keys = ShareUsageEvidenceKeyFilter(query_whitelist=["utm_source"])
+ keys = ShareUsageEvidenceKeyFilter()
+ pipeline = UsingPipeline("query.utm_source")
- self.assertTrue(keys.filter_evidence_key("query.utm_source"))
- self.assertFalse(keys.filter_evidence_key("server.request-query"))
+ self.assertTrue(keys.filter_evidence_key("query.utm_source", pipeline))
+ self.assertFalse(
+ keys.filter_evidence_key("server.request-query", pipeline))
diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py
index 9209ae53..b31b9f1f 100644
--- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py
+++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py
@@ -21,8 +21,9 @@
# *********************************************************************
"""A configuration file gives a list of names as one comma-separated string,
-so the usage sharing element takes its query string parameters to share and
-its headers not to share in that form, as well as in a list."""
+so the usage sharing element takes its headers not to share in that form, as
+well as in a list. Its deprecated query string parameters to share are read
+the same way and shared no more than any other parameter."""
import json
import unittest
@@ -44,8 +45,7 @@
"endpoint": "http://127.0.0.1:9",
}
-# Shares only the query string parameters listed and those starting 51D_,
-# where every one is shared by default.
+# The deprecated option, which no longer changes what is shared.
LISTED_ONLY = {"share_all_query_string_parameters": False}
# A configuration file that gives both lists as comma-separated strings.
@@ -114,8 +114,9 @@ def test_query_parameters_to_share_from_a_string(self):
ShareUsage(query_whitelist="first,second", **LISTED_ONLY,
**QUEUE_ONLY))
- self.assert_shared(shared, ("first value", "second value"))
- self.assert_not_shared(shared, ("third value", "letter value"))
+ self.assert_not_shared(
+ shared,
+ ("first value", "second value", "third value", "letter value"))
def test_headers_not_to_share_from_a_string(self):
shared = shared_alone_by(
@@ -130,10 +131,11 @@ def test_lists_from_a_configuration_file(self):
usage = pipeline.get_element("shareusage")
shared = shared_by(pipeline, usage)
- self.assert_shared(
- shared, ("first value", "second value", "agent value"))
+ self.assertEqual(["first", "second"], usage.query_whitelist)
+ self.assert_shared(shared, ("agent value",))
self.assert_not_shared(
- shared, ("third value", "letter value", "one value", "two value"))
+ shared, ("first value", "second value", "third value",
+ "letter value", "one value", "two value"))
def test_names_are_trimmed_and_empty_entries_left_out(self):
usage = ShareUsage(
@@ -165,10 +167,10 @@ def test_a_list_is_used_as_it_is(self):
self.assertIs(names, usage.query_whitelist)
self.assertIs(headers, usage.header_blacklist)
shared = shared_alone_by(usage)
- self.assert_shared(
- shared, ("first value", "second value", "agent value"))
+ self.assert_shared(shared, ("agent value",))
self.assert_not_shared(
- shared, ("third value", "letter value", "one value", "two value"))
+ shared, ("first value", "second value", "third value",
+ "letter value", "one value", "two value"))
if __name__ == "__main__":
diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py
index 5df7e9be..40a4bd51 100644
--- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py
+++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py
@@ -21,10 +21,11 @@
# *********************************************************************
"""
-What usage sharing shares by default and when all Evidence is shared.
-Every query string and form parameter is shared unless that is turned off,
-no cookie but a 51D_ one is ever shared, and the session cookie is still
-compared when sessions are tracked.
+What usage sharing shares by default and when all Evidence is shared. A query
+string or form parameter is shared only when its name starts with 51D_ or the
+Pipeline uses it as evidence, whatever the settings, no cookie but a 51D_ one
+is ever shared, and the session cookie is still compared when sessions are
+tracked.
"""
import gzip
@@ -36,6 +37,8 @@
import requests
from flask import Flask, request
+from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \
+ BasicListEvidenceKeyFilter
from fiftyone_pipeline_core.flowelement import FlowElement
from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder
from fiftyone_pipeline_core.web import webevidence
@@ -53,8 +56,8 @@
"endpoint": "http://127.0.0.1:9",
}
-# Turns off sharing every query string parameter, from a configuration
-# file.
+# The deprecated option to share every query string parameter, from a
+# configuration file.
CONFIGURATION = """
{
"PipelineOptions": {
@@ -104,50 +107,91 @@ def records_of(usage, requests_evidence, *elements):
return sent_on_close(usage)
-class QueryAndFormValuesTests(unittest.TestCase):
+class UsingElement(FlowElement):
+ """Accepts the query string parameter given as evidence, and adds no
+ data."""
+
+ def __init__(self, key):
+ super(UsingElement, self).__init__()
+ self.datakey = "using"
+ self.key = key
+
+ def get_evidence_key_filter(self):
+ return BasicListEvidenceKeyFilter([self.key])
+
+
+# Every way the deprecated options could once share more.
+DEPRECATED_SETTINGS = (
+ {},
+ {"share_all": True},
+ {"query_whitelist": ["password", "campaign"]},
+ {"share_all_query_string_parameters": True},
+ {"share_all_query_string_parameters": False,
+ "query_whitelist": "password,campaign"},
+)
+
+
+def shared_from_web_request(settings, method):
+ """The record usage sharing sends for one web request with the
+ parameters below, in its query string for GET or its form body for
+ POST, through a Pipeline whose other element uses query.used."""
+ app = Flask(__name__)
+ usage = ShareUsage(**dict(QUEUE_ONLY, **settings))
+ pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS) \
+ .add(UsingElement("query.used")).add(usage).build()
+ parameters = {"51D_screen": "fiftyone value", "used": "used value",
+ "password": "password value", "campaign": "campaign value"}
+ if method == "GET":
+ context = app.test_request_context("/", query_string=parameters)
+ else:
+ context = app.test_request_context(
+ "/", method="POST", data=parameters)
+ with context:
+ flowdata = pipeline.create_flowdata()
+ flowdata.evidence.add_from_dict(webevidence(request))
+ flowdata.process()
+ records = sent_on_close(usage)
+ return records[0] if records else ""
- def test_every_query_value_is_shared_by_default(self):
- records = records_of(ShareUsage(**QUEUE_ONLY), [{
- "header.user-agent": "agent value",
- "query.campaign": "campaign value",
- "query.password": "password value",
- }])
- self.assertEqual(1, len(records))
- self.assertIn('campaign value',
- records[0])
- self.assertIn('password value',
- records[0])
+class QueryAndFormValuesTests(unittest.TestCase):
- def test_every_form_value_of_the_web_request_is_shared_by_default(self):
- app = Flask(__name__)
+ def test_a_51d_parameter_is_shared_from_a_query_string_and_a_form(self):
+ for method in ("GET", "POST"):
+ with self.subTest(method=method):
+ record = shared_from_web_request({}, method)
+ self.assertIn("fiftyone value", record)
+
+ def test_a_parameter_the_pipeline_uses_is_shared_from_both(self):
+ for method in ("GET", "POST"):
+ with self.subTest(method=method):
+ record = shared_from_web_request({}, method)
+ self.assertIn('used value', record)
+
+ def test_no_other_parameter_is_ever_shared_from_either(self):
+ for settings in DEPRECATED_SETTINGS:
+ for method in ("GET", "POST"):
+ with self.subTest(settings=settings, method=method):
+ record = shared_from_web_request(settings, method)
+ self.assertIn("fiftyone value", record)
+ self.assertNotIn("password value", record)
+ self.assertNotIn("campaign value", record)
+
+ def test_the_javascript_builder_uses_only_the_parameters_it_lists(self):
usage = ShareUsage(**QUEUE_ONLY)
- pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS).add(usage).build()
- with app.test_request_context(
- "/", method="POST", data={"field": "form value"}):
- flowdata = pipeline.create_flowdata()
- flowdata.evidence.add_from_dict(webevidence(request))
- flowdata.process()
+ pipeline = PipelineBuilder().add(usage).build()
+ flowdata = pipeline.create_flowdata()
+ flowdata.evidence.add("header.user-agent", "agent value")
+ flowdata.evidence.add("query.password", "password value")
+ flowdata.evidence.add("query.fod-js-object-name", "object value")
+ flowdata.process()
records = sent_on_close(usage)
- self.assertEqual(1, len(records))
- self.assertIn('form value', records[0])
-
- def test_turned_off_only_51d_and_listed_values_are_shared(self):
- records = records_of(ShareUsage(
- share_all_query_string_parameters=False,
- query_whitelist=["campaign"], **QUEUE_ONLY), [{
- "header.user-agent": "agent value",
- "query.campaign": "campaign value",
- "query.51d_gethighentropyvalues": "hev",
- "query.password": "password value",
- }])
-
- self.assertIn("campaign value", records[0])
- self.assertIn("hev", records[0])
+ self.assertIn("agent value", records[0])
+ self.assertIn("object value", records[0])
self.assertNotIn("password value", records[0])
- def test_turned_off_from_a_configuration_file(self):
+ def test_the_deprecated_option_from_a_configuration_file(self):
pipeline = PipelineBuilder().build_from_configuration(
json.loads(CONFIGURATION))
usage = pipeline.get_element("shareusage")
@@ -161,7 +205,6 @@ def test_turned_off_from_a_configuration_file(self):
self.assertIn("agent value", records[0])
self.assertNotIn("password value", records[0])
-
class CookieTests(unittest.TestCase):
def test_only_51d_cookies_are_shared_when_all_evidence_is_shared(self):
diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py
index c415387e..c3749389 100644
--- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py
+++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py
@@ -38,6 +38,22 @@
import requests
from unittest import mock
+from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \
+ BasicListEvidenceKeyFilter
+from fiftyone_pipeline_core.flowelement import FlowElement
+
+
+class UsingElement(FlowElement):
+ """Accepts the evidence keys given, and adds no data."""
+
+ def __init__(self, keys):
+ super(UsingElement, self).__init__()
+ self.datakey = "using"
+ self.keys = keys
+
+ def get_evidence_key_filter(self):
+ return BasicListEvidenceKeyFilter(self.keys)
+
testHost = "127.0.0.1"
testPort = 5000
testEndpoint = f"http://{testHost}:{testPort}"
@@ -919,9 +935,9 @@ def test_escapes_xml_characters(self):
def test_default_shares_only_accepted_evidence(self):
"""!
- Check that with no options the element shares headers, the
- 51Degrees cookies and every query string and form parameter, and
- leaves out other cookies, the Cookie header and the email address.
+ Check that with no options the element shares headers and the
+ 51Degrees cookies and parameters, and leaves out other cookies, the
+ Cookie header, and other query string and form parameters.
"""
usage = ShareUsage(
@@ -948,7 +964,7 @@ def test_default_shares_only_accepted_evidence(self):
self.assertIn('hev', app.received[0])
self.assertNotIn("cookie value", app.received[0])
self.assertNotIn("header value", app.received[0])
- self.assertIn('password value', app.received[0])
+ self.assertNotIn("password value", app.received[0])
self.assertNotIn("email value", app.received[0])
def test_never_shared_evidence(self):
@@ -984,8 +1000,9 @@ def test_never_shared_evidence(self):
def test_share_all(self):
"""!
Check that sharing all evidence shares what the other rules leave
- out, but not the evidence that is never shared or a cookie that
- does not start 51D_.
+ out, but not the evidence that is never shared, a cookie that
+ does not start 51D_, or a parameter that neither starts 51D_ nor is
+ used by the Pipeline.
"""
usage = ShareUsage(
@@ -1007,22 +1024,21 @@ def test_share_all(self):
self.assertEqual(len(app.received), 1)
self.assertNotIn("cookie value", app.received[0])
self.assertIn('1080', app.received[0])
- self.assertIn('campaign value', app.received[0])
+ self.assertNotIn("campaign value", app.received[0])
self.assertNotIn("email value", app.received[0])
def test_parameter_names_with_dots(self):
"""!
Check that a query string parameter whose name contains a dot is
- matched by its whole name against the parameters to share.
+ matched by its whole name against the evidence the Pipeline uses.
"""
usage = ShareUsage(
requested_package_size=1,
- endpoint=testEndpoint,
- share_all_query_string_parameters=False,
- query_whitelist=["valid.email"])
+ endpoint=testEndpoint)
pipeline = (PipelineBuilder())\
+ .add(UsingElement(["query.valid.email"]))\
.add(usage)\
.build()
flowdata = pipeline.create_flowdata()