Publish QTI editor formulas as MathML

contentcuration/contentcuration/frontend/shared/views/QTIEditor/serialization/__tests__/formulaItem.spec.js

@@ -115,3 +115,58 @@ it('keeps the text after a formula in a hint once loaded into the editor', () =>
   const item = buildItem(INTERACTIONS[0]);
   expect(load(item.hints[0].content)).toBe(CONTENT);
 });
+
+describe('an item converted from legacy markdown with formulas', () => {
+  const inlineMath = (latex, mathml) =>
+    '<math xmlns="http://www.w3.org/1998/Math/MathML" display="inline"><semantics>' +
+    `<mrow>${mathml}</mrow><annotation encoding="application/x-tex">${latex}</annotation>` +
+    '</semantics></math>';
+  const blockMath = (latex, mathml) =>
+    inlineMath(latex, mathml).replace('display="inline"', 'display="block"');
+  const RAW_DATA =
+    '<qti-assessment-item xmlns="http://www.imsglobal.org/xsd/imsqtiasi_v3p0" ' +
+    'identifier="item_1" title="Question" xml:lang="en">' +
+    '<qti-response-declaration identifier="RESPONSE" cardinality="single" ' +
+    'base-type="identifier"><qti-correct-response><qti-value>choice_0</qti-value>' +
+    '</qti-correct-response></qti-response-declaration>' +
+    '<qti-item-body><qti-choice-interaction response-identifier="RESPONSE" ' +
+    'max-choices="1"><qti-prompt>' +
+    `<p>What is ${inlineMath('x^2 &lt; 1', '<mi>x</mi>')} when x is 2?</p>` +
+    blockMath('\\frac{a}{b}', '<mi>a</mi>') +
+    '</qti-prompt><qti-simple-choice identifier="choice_0">' +
+    blockMath('\\frac{1}{2}', '<mn>1</mn>') +
+    '</qti-simple-choice><qti-simple-choice identifier="choice_1">' +
+    '<p>Other</p></qti-simple-choice></qti-choice-interaction></qti-item-body>' +
+    '<qti-catalog-info><qti-catalog id="kolibri-hints"><qti-card support="ext:kolibri-hint">' +
+    `<qti-html-content><p>Square it: ${inlineMath('2^2', '<mn>2</mn>')} first</p>` +
+    '</qti-html-content></qti-card></qti-catalog></qti-catalog-info></qti-assessment-item>';
+
+  let item;
+  let state;
+
+  beforeAll(() => {
+    item = parseItem(RAW_DATA);
+    const [interaction] = item.interactions;
+    state = choiceInteractionDescriptor.parse(
+      interaction.bodyXml,
+      interaction.responseDeclarations,
+    );
+  });
+
+  it('loads inline and block formulas in the prompt as editor formulas', () => {
+    expect(load(state.prompt)).toBe(
+      '<p>What is <span data-latex="x^2 < 1"></span> when x is 2?</p>' +
+        '<p><span data-latex="\\frac{a}{b}"></span></p>',
+    );
+  });
+
+  it('loads a formula answer as an editor formula', () => {
+    expect(load(state.choices[0].content)).toBe('<p><span data-latex="\\frac{1}{2}"></span></p>');
+  });
+
+  it('loads a formula in a hint as an editor formula', () => {
+    expect(load(item.hints[0].content)).toBe(
+      '<p>Square it: <span data-latex="2^2"></span> first</p>',
+    );
+  });
+});

contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Math.js

@@ -2,6 +2,14 @@ import { Node, mergeAttributes } from '@tiptap/core';
 import { VueNodeViewRenderer } from '@tiptap/vue-2';
 import LazyMathNodeView from '../components/math/LazyMathNodeView.vue';
 
+/**
+ * @param {Element} math - A MathML `<math>` element
+ * @returns {string|undefined} The LaTeX it was rendered from, as published formulas annotate it
+ */
+export function mathmlLatex(math) {
+  return math.querySelector('annotation[encoding="application/x-tex"]')?.textContent;
+}
+
 export const Math = Node.create({
   name: 'math',
   group: 'inline',
@@ -23,7 +31,16 @@ export const Math = Node.create({
   },
 
   parseHTML() {
-    return [{ tag: 'span[data-latex]' }];
+    return [
+      { tag: 'span[data-latex]' },
+      {
+        tag: 'math',
+        getAttrs: element => {
+          const latex = mathmlLatex(element);
+          return latex ? { latex } : false;
+        },
+      },
+    ];
   },
 
   renderHTML({ HTMLAttributes }) {

contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/comparisonKey.js

@@ -4,6 +4,7 @@ import { getSchema } from '@tiptap/core';
 // eslint-disable-next-line import/no-unresolved
 import { DOMParser as PMDOMParser } from '@tiptap/pm/model';
 import { editorExtensions } from '../extensions';
+import { mathmlLatex } from '../extensions/Math';
 
 // The full schema, so inline and block content of the same text read the same.
 const schema = getSchema(editorExtensions());
@@ -60,7 +61,7 @@ export function hasRichTextContent(content) {
  *
  * Its text, however marked up except its sub/superscripts, and each formula, image,
  * SVG, MathML and line break where it stands. Read as TipTap saves it, so its
- * re-render keys the same, unless it has SVG or MathML, which TipTap drops.
+ * re-render keys the same, unless it has SVG or MathML without its LaTeX, which TipTap drops.
  *
  * @param {string} html - HTML fragment from a TipTap editor
  * @returns {string} '' for a fragment without rich text content
@@ -72,6 +73,7 @@ export function richTextComparisonKey(html) {
   // into U+FFFD, so no stored formula has this LaTeX.
   const unparsed = new Map();
   for (const element of document.body.querySelectorAll(UNPARSED_MEDIA)) {
+    if (element.localName === 'math' && mathmlLatex(element)) continue;
     const latex = `\0${unparsed.size}`;
     unparsed.set(latex, ['html', element.outerHTML.replace(/\s+/g, ' ').replace(/>\s+</g, '><')]);
     const placeholder = document.createElement('span');

contentcuration/contentcuration/frontend/shared/views/TipTapEditor/__tests__/comparisonKey.spec.js

@@ -205,6 +205,9 @@ describe('richTextComparisonKey', () => {
       '<pre><code>if x:\n    return 1</code></pre>',
       '<p>a<br>b</p>',
       '<p>a <br>\n  b</p>',
+      '<p>Solve <math xmlns="http://www.w3.org/1998/Math/MathML" display="inline"><semantics>' +
+        '<mi>x</mi><annotation encoding="application/x-tex">x</annotation></semantics></math>' +
+        ' now</p>',
     ])('%j keys the same', stored => {
       expect(richTextComparisonKey(rewrap(stored))).toBe(richTextComparisonKey(stored));
     });

contentcuration/contentcuration/tests/test_restore_channel.py

@@ -447,6 +447,22 @@ def test_qti_only_node_restores_editor_and_legacy_items_as_qti(self):
         self.assertIn("Legacy question", items[1].raw_data)
         self._assert_republishes_source_package(QTIExerciseGenerator)
 
+    def test_editor_item_formula_restores_as_published_mathml(self):
+        item = self._add_editor_item(0)
+        item.raw_data = item.raw_data.replace(
+            "Pick one.", 'Pick <span data-latex="x^2"></span> one.'
+        )
+        item.save()
+        self._publish(QTIExerciseGenerator)
+
+        items = self._restore(format_presets.QTI_ZIP)
+
+        self.assertIn(
+            '<annotation encoding="application/x-tex">x^2</annotation>',
+            items[0].raw_data,
+        )
+        self._assert_republishes_source_package(QTIExerciseGenerator)
+
     def test_editor_item_with_non_canonical_id_gets_a_new_assessment_id(self):
         # Matches hex_to_qti_id's shape but sets its padding bits.
         identifier = "K" + "A" * 21 + "B"

contentcuration/contentcuration/tests/utils/qti/fixtures/free_response_with_maths.xml

@@ -8,7 +8,7 @@
     <qti-outcome-declaration identifier="SCORE" cardinality="single" base-type="float" />
     <qti-item-body>
         <div>
-            <math display="block">
+            <math xmlns="http://www.w3.org/1998/Math/MathML" display="block">
                 <semantics>
                     <mrow>
                         <munderover><mo>∑</mo><mi>n</mi><mi>s</mi></munderover>

contentcuration/contentcuration/tests/utils/qti/fixtures/math_content_choice_interaction.xml

@@ -9,7 +9,7 @@
     <qti-item-body>
         <qti-choice-interaction response-identifier="RESPONSE" shuffle="true" max-choices="1" min-choices="0" orientation="vertical">
             <qti-prompt>
-                <p>Solve the equation <math display="inline"><semantics><mrow><mfrac><mrow><mi>x</mi></mrow><mrow><mn>2</mn></mrow></mfrac><mo>=</mo><mn>3</mn></mrow><annotation encoding="application/x-tex">\frac{x}{2} = 3</annotation></semantics></math> for x. What is the value of x?</p>
+                <p>Solve the equation <math xmlns="http://www.w3.org/1998/Math/MathML" display="inline"><semantics><mrow><mfrac><mrow><mi>x</mi></mrow><mrow><mn>2</mn></mrow></mfrac><mo>=</mo><mn>3</mn></mrow><annotation encoding="application/x-tex">\frac{x}{2} = 3</annotation></semantics></math> for x. What is the value of x?</p>
             </qti-prompt>
             <qti-simple-choice identifier="choice_0" show-hide="show" fixed="false"><p>6</p></qti-simple-choice>
             <qti-simple-choice identifier="choice_1" show-hide="show" fixed="false"><p>3</p></qti-simple-choice>

contentcuration/contentcuration/tests/utils/qti/test_convert.py

@@ -213,6 +213,7 @@ def test_math_content_in_choice_interaction(self):
             _normalize_xml(_load_fixture("math_content_choice_interaction.xml")),
             _normalize_xml(result.xml),
         )
+        self.assertTrue(validate_qti_item(result.xml.encode("utf-8")).is_valid)
 
 
 class TextEntryInteractionConversionTests(unittest.TestCase):
@@ -347,6 +348,7 @@ def test_free_response_with_maths(self):
             _normalize_xml(_load_fixture("free_response_with_maths.xml")),
             _normalize_xml(result.xml),
         )
+        self.assertTrue(validate_qti_item(result.xml.encode("utf-8")).is_valid)
 
 
 class MarkdownContentConversionTests(unittest.TestCase):
@@ -483,13 +485,11 @@ def test_a_space_before_inline_math_survives(self):
         # inline all the same, so the gap before it is a gap the reader sees.
         result = self._convert("It is ~~wrong~~ $$x$$ here.")
 
-        # No validity assertion here: inline math does not validate against the item
-        # XSD on unstable either, because the <math> is serialized into the QTI
-        # namespace rather than the MathML one. That is its own bug, not this gap.
         self.assertIn(
             '<p>It is <span style="text-decoration: line-through;">wrong</span> <math',
             result.xml,
         )
+        self.assertTrue(validate_qti_item(result.xml.encode("utf-8")).is_valid)
 
     def test_a_space_before_an_inline_image_survives(self):
         result = self._convert("It is ~~wrong~~ ![i](x.png) here.")
@@ -569,21 +569,39 @@ def test_raw_html_highlight_is_stripped(self):
         self.assertTrue(validate_qti_item(result.xml.encode("utf-8")).is_valid)
 
     def test_maths_in_a_list_item(self):
-        # Validity is not asserted here or below: rendered MathML carries no
-        # namespace, the same gap test_free_response_with_maths lives with.
         result = self._convert("- $$x^2$$\n- two")
 
-        self.assertIn('<li><math display="block">', result.xml)
+        self.assertIn(
+            '<li><math xmlns="http://www.w3.org/1998/Math/MathML" display="block">',
+            result.xml,
+        )
+        self.assertTrue(validate_qti_item(result.xml.encode("utf-8")).is_valid)
 
     def test_maths_in_a_table_cell(self):
         result = self._convert("| a |\n|---|\n| $$x^2$$ |")
 
-        self.assertIn('<td><math display="inline">', result.xml)
+        self.assertIn(
+            '<td><math xmlns="http://www.w3.org/1998/Math/MathML" display="inline">',
+            result.xml,
+        )
+        self.assertTrue(validate_qti_item(result.xml.encode("utf-8")).is_valid)
+
+    def test_maths_latex2mathml_rejects_converts_to_a_valid_item(self):
+        for markdown in ("It is $$x_$$ here.", "$${$$", "- $$a<b_$$\n- two"):
+            with self.subTest(markdown=markdown):
+                result = self._convert(markdown)
+
+                self.assertNotIn("<math", result.xml)
+                self.assertTrue(validate_qti_item(result.xml.encode("utf-8")).is_valid)
 
     def test_maths_in_an_answer(self):
         result = self._convert(answers=[{"answer": "$$x^2$$", "correct": True}])
 
-        self.assertIn('><math display="block">', result.xml)
+        self.assertIn(
+            '><math xmlns="http://www.w3.org/1998/Math/MathML" display="block">',
+            result.xml,
+        )
+        self.assertTrue(validate_qti_item(result.xml.encode("utf-8")).is_valid)
 
 
 class CustomInteractionTests(unittest.TestCase):

contentcuration/contentcuration/tests/utils/qti/test_html_to_markdown.py

@@ -97,6 +97,13 @@ def test_underlined_span_becomes_a_perseus_underline():
     )
 
 
+def test_formula_span_becomes_double_dollar_latex():
+    assert (
+        _markdown_from_html('<p>Is <span data-latex="x^2"></span> even?</p>')
+        == "Is $$x^2$$ even?"
+    )
+
+
 def test_span_carrying_both_decorations_keeps_both():
     assert (
         _markdown_from_html(

contentcuration/contentcuration/tests/utils/qti/test_mathml.py

@@ -636,7 +636,10 @@ def test_math_element_with_attributes(self):
             children=[Mi(children=["y"])],
         )
         xml_str = obj.to_xml_string()
-        expected_xml_str = '<math display="block" alttext="Equation"><mi>y</mi></math>'
+        expected_xml_str = (
+            '<math xmlns="http://www.w3.org/1998/Math/MathML" display="block"'
+            ' alttext="Equation"><mi>y</mi></math>'
+        )
         self.assertEqual(xml_str, expected_xml_str)
 
     def test_mrow_nested_elements(self):
@@ -899,6 +902,12 @@ def test_from_string_unregistered_tag(self):
         ):
             MathMLElement.from_string(xml_str)
 
+    def test_from_string_foreign_namespace_tag(self):
+        with self.assertRaisesRegex(
+            ValueError, "No registered class found for tag: {urn:foo}mi"
+        ):
+            MathMLElement.from_string('<mi xmlns="urn:foo">x</mi>')
+
     def test_attribute_validation_error_on_creation(self):
         with self.assertRaises(ValueError):  # Pydantic's ValidationError
             Mi(mathvariant="not-a-valid-variant", children=["x"])

contentcuration/contentcuration/tests/utils/qti/test_media.py

@@ -1,5 +1,11 @@
+from lxml import etree
+
+from contentcuration.tests.utils.qti.test_validation import SINGLE_HINT_CATALOG
 from contentcuration.tests.utils.qti.test_validation import VALID_CHOICE_ITEM
+from contentcuration.utils.assessment.markdown import latex_to_mathml
+from contentcuration.utils.assessment.qti.constants import MATHML_NAMESPACE
 from contentcuration.utils.assessment.qti.media import get_qti_media_references
+from contentcuration.utils.assessment.qti.media import render_qti_latex_spans
 from contentcuration.utils.assessment.qti.media import rewrite_qti_media_paths
 from contentcuration.utils.assessment.qti.media import rewrite_qti_sized_image_paths
 from contentcuration.utils.assessment.qti.media import set_qti_item_language
@@ -289,3 +295,58 @@ def test_strip_studio_attributes_ignores_lookalikes_in_values_and_text():
         "data-other": "1",
     }
     assert paragraph.text == 'data-studio-prompt=""'
+
+
+def _with_latex_spans(prompt, hint):
+    return VALID_CHOICE_ITEM.replace(
+        "<qti-prompt>Select the correct answer.</qti-prompt>",
+        f"<qti-prompt>{prompt}</qti-prompt>",
+    ).replace(
+        "<qti-response-processing",
+        SINGLE_HINT_CATALOG.format(hint=hint) + "<qti-response-processing",
+    )
+
+
+def _canonical(xml):
+    return etree.tostring(_root(xml), method="c14n")
+
+
+def test_render_latex_spans_puts_mathml_in_the_prompt_and_a_hint():
+    span = '<span data-latex="x^2"></span>'
+    result = render_qti_latex_spans(
+        _with_latex_spans(f"Solve {span} now.", f"Try {span} again.")
+    )
+    mathml = latex_to_mathml("x^2")
+    assert _canonical(result) == _canonical(
+        _with_latex_spans(f"Solve {mathml} now.", f"Try {mathml} again.")
+    )
+    validation = validate_qti_item(result)
+    assert validation.is_valid, validation.errors
+
+
+def test_render_latex_spans_escapes_markup_characters_in_the_annotation():
+    result = render_qti_latex_spans(
+        _with_latex_spans(
+            '<span data-latex="\\begin{matrix}a &lt; b &amp; c\\end{matrix}"></span>',
+            "Hint.",
+        )
+    )
+    annotation = _root(result).find(f".//{{{MATHML_NAMESPACE}}}annotation")
+    assert annotation.text == r"\begin{matrix}a < b & c\end{matrix}"
+    validation = validate_qti_item(result)
+    assert validation.is_valid, validation.errors
+
+
+def test_render_latex_spans_leaves_malformed_latex_as_its_source_text():
+    result = render_qti_latex_spans(
+        _with_latex_spans('Solve <span data-latex="x_"></span> now.', "Hint.")
+    )
+    prompt = _root(result).find(".//{*}qti-prompt")
+    assert list(prompt) == []
+    assert prompt.text == "Solve $$x_$$ now."
+    validation = validate_qti_item(result)
+    assert validation.is_valid, validation.errors
+
+
+def test_render_latex_spans_leaves_an_item_without_formulas_unchanged():
+    assert render_qti_latex_spans(VALID_CHOICE_ITEM) == VALID_CHOICE_ITEM

contentcuration/contentcuration/tests/utils/qti/test_validation.py

@@ -47,6 +47,13 @@ def _item_xml(identifier, title, response_declaration, item_body):
     "</qti-choice-interaction>",
 )
 
+SINGLE_HINT_CATALOG = (
+    '<qti-catalog-info><qti-catalog id="kolibri-hints">'
+    '<qti-card support="ext:kolibri-hint">'
+    "<qti-html-content><p>{hint}</p></qti-html-content>"
+    "</qti-card></qti-catalog></qti-catalog-info>"
+)
+
 _XML_DECLARATION = '<?xml version="1.0" encoding="UTF-8"?>'
 
 _ENTITY_DOCTYPE = '<!DOCTYPE qti-assessment-item [<!ENTITY e "hi">]>'