From 93d9fbf52013721049ae0c108bdabad4735e9544 Mon Sep 17 00:00:00 2001 From: rtibblesbot Date: Tue, 27 Oct 2026 15:53:16 -0700 Subject: [PATCH 1/7] Put converted formulas in the MathML namespace Rendered now declares the MathML namespace, so converted items with formulas pass the item XSD. Renames _convert to latex_to_mathml. Co-Authored-By: Claude Opus 5.5 --- .../qti/fixtures/free_response_with_maths.xml | 2 +- .../math_content_choice_interaction.xml | 2 +- .../tests/utils/qti/test_convert.py | 26 +++++++--- .../tests/utils/qti/test_mathml.py | 11 +++- .../tests/utils/test_markdown.py | 50 ++++++++++++------- .../utils/assessment/markdown.py | 27 +++++++--- .../utils/assessment/qti/base.py | 4 +- .../utils/assessment/qti/constants.py | 2 + .../utils/assessment/qti/mathml/core.py | 3 +- 9 files changed, 90 insertions(+), 37 deletions(-) diff --git a/contentcuration/contentcuration/tests/utils/qti/fixtures/free_response_with_maths.xml b/contentcuration/contentcuration/tests/utils/qti/fixtures/free_response_with_maths.xml index 62600279de..19d9e79ae7 100644 --- a/contentcuration/contentcuration/tests/utils/qti/fixtures/free_response_with_maths.xml +++ b/contentcuration/contentcuration/tests/utils/qti/fixtures/free_response_with_maths.xml @@ -8,7 +8,7 @@
- + ∑ns diff --git a/contentcuration/contentcuration/tests/utils/qti/fixtures/math_content_choice_interaction.xml b/contentcuration/contentcuration/tests/utils/qti/fixtures/math_content_choice_interaction.xml index f4fce7ac35..3d7efb2612 100644 --- a/contentcuration/contentcuration/tests/utils/qti/fixtures/math_content_choice_interaction.xml +++ b/contentcuration/contentcuration/tests/utils/qti/fixtures/math_content_choice_interaction.xml @@ -9,7 +9,7 @@ -

Solve the equation x2=3\frac{x}{2} = 3 for x. What is the value of x?

+

Solve the equation x2=3\frac{x}{2} = 3 for x. What is the value of x?

6

3

 diff --git a/contentcuration/contentcuration/tests/utils/qti/test_convert.py b/contentcuration/contentcuration/tests/utils/qti/test_convert.py index 83bc884f70..59fb2bfe06 100644 --- a/contentcuration/contentcuration/tests/utils/qti/test_convert.py +++ b/contentcuration/contentcuration/tests/utils/qti/test_convert.py @@ -213,6 +213,7 @@ def test_math_content_in_choice_interaction(self): _normalize_xml(_load_fixture("math_content_choice_interaction.xml")), _normalize_xml(result.xml), ) + self.assertTrue(validate_qti_item(result.xml.encode("utf-8")).is_valid) class TextEntryInteractionConversionTests(unittest.TestCase): @@ -347,6 +348,7 @@ def test_free_response_with_maths(self): _normalize_xml(_load_fixture("free_response_with_maths.xml")), _normalize_xml(result.xml), ) + self.assertTrue(validate_qti_item(result.xml.encode("utf-8")).is_valid) class MarkdownContentConversionTests(unittest.TestCase): @@ -483,13 +485,11 @@ def test_a_space_before_inline_math_survives(self): # inline all the same, so the gap before it is a gap the reader sees. result = self._convert("It is ~~wrong~~ $$x$$ here.") - # No validity assertion here: inline math does not validate against the item - # XSD on unstable either, because the is serialized into the QTI - # namespace rather than the MathML one. That is its own bug, not this gap. self.assertIn( '

It is wrong ', result.xml) + self.assertIn( + '

  • ', + result.xml, + ) + self.assertTrue(validate_qti_item(result.xml.encode("utf-8")).is_valid) def test_maths_in_a_table_cell(self): result = self._convert("| a |\n|---|\n| $$x^2$$ |") - self.assertIn('', result.xml) + self.assertIn( + '', + result.xml, + ) + self.assertTrue(validate_qti_item(result.xml.encode("utf-8")).is_valid) def test_maths_in_an_answer(self): result = self._convert(answers=[{"answer": "$$x^2$$", "correct": True}]) - self.assertIn('>', result.xml) + self.assertIn( + '>', + result.xml, + ) + self.assertTrue(validate_qti_item(result.xml.encode("utf-8")).is_valid) class CustomInteractionTests(unittest.TestCase): diff --git a/contentcuration/contentcuration/tests/utils/qti/test_mathml.py b/contentcuration/contentcuration/tests/utils/qti/test_mathml.py index 0bace05336..340ce6129e 100644 --- a/contentcuration/contentcuration/tests/utils/qti/test_mathml.py +++ b/contentcuration/contentcuration/tests/utils/qti/test_mathml.py @@ -636,7 +636,10 @@ def test_math_element_with_attributes(self): children=[Mi(children=["y"])], ) xml_str = obj.to_xml_string() - expected_xml_str = 'y' + expected_xml_str = ( + 'y' + ) self.assertEqual(xml_str, expected_xml_str) def test_mrow_nested_elements(self): @@ -899,6 +902,12 @@ def test_from_string_unregistered_tag(self): ): MathMLElement.from_string(xml_str) + def test_from_string_foreign_namespace_tag(self): + with self.assertRaisesRegex( + ValueError, "No registered class found for tag: {urn:foo}mi" + ): + MathMLElement.from_string('x') + def test_attribute_validation_error_on_creation(self): with self.assertRaises(ValueError): # Pydantic's ValidationError Mi(mathvariant="not-a-valid-variant", children=["x"]) diff --git a/contentcuration/contentcuration/tests/utils/test_markdown.py b/contentcuration/contentcuration/tests/utils/test_markdown.py index da234df9e7..d2aa56dbcc 100644 --- a/contentcuration/contentcuration/tests/utils/test_markdown.py +++ b/contentcuration/contentcuration/tests/utils/test_markdown.py @@ -19,7 +19,7 @@ def test_markdown_with_inline_math(self): ) expected = ( "

    What is the answer to this question? " - '' + '' "x·y=z2" 'x\cdot y=z^2' # noqa W605 "

    \n" @@ -35,7 +35,7 @@ def test_block_math(self): ) expected = ( "

    Here's an equation:

    \n" - '' + '' "E=mc2" 'E = mc^2' "" @@ -59,7 +59,7 @@ def test_multiline_block_math(self): "$$" ) expected = ( - '' + '' "a=b+c" '$5=d+e' '\n\\begin{aligned}\na = b + c \\\\\n$5 = d + e\n\\end{aligned}\n' @@ -77,7 +77,7 @@ def test_inline_math_with_dollar_inside(self): markdown_text = "Test this $$\\text{Cost = 1.00 $USD$}$$" expected = ( "

    Test this " - '' + '' "Cost = 1.00 $USD$" '\\text{Cost = 1.00 $USD$}' "

    \n" @@ -85,19 +85,34 @@ def test_inline_math_with_dollar_inside(self): self._assert_conversion(markdown_text, expected) + def test_text_with_markup_characters(self): + markdown_text = "$$\\text{a' + "a<b+" + "c&d+" + "e &lt; f" + '' + "\\text{a<b}+\\text{c&d}+\\text{e &lt; f}" + "" + "" + ) + + self._assert_conversion(markdown_text, expected) + def test_multiple_math_expressions(self): """Test multiple math expressions in one document""" markdown_text = "First: $$a + b$$, then $$c \\times d$$, finally $$e^f$$." expected = ( "

    First: " - 'a+b' + 'a+b' 'a + b' ", then " - 'c×d' + 'c×d' 'c \\times d' ", finally " - 'ef' + 'ef' 'e^f' ".

    \n" ) @@ -115,17 +130,17 @@ def test_mixed_inline_and_block(self): ) expected = ( "

    This is inline math: " - 'a=b' + 'a=b' 'a = b' "

    \n" "

    And this is block math:

    \n" - '' + '' "∑i=1" "nxi=y" '\sum_{i=1}^{n} x_i = y' # noqa W605 "" "

    Back to text with more inline: " - 'z2' + 'z2' 'z^2' "

    \n" ) @@ -137,7 +152,7 @@ def test_block_sum_uses_munderover(self): markdown_text = "$$\\sum_{i=1}^{n} x_i$$" expected = ( - '' + '' "∑i=1" "nxi" '\\sum_{i=1}^{n} x_i' @@ -152,7 +167,8 @@ def test_inline_sum_uses_msubsup(self): markdown_text = "The sum $$\\sum_{i=1}^{n} x_i$$ is finite." expected = ( "

    The sum " - '∑i=1' + '' + "∑i=1" "nxi" '\\sum_{i=1}^{n} x_i' " is finite.

    \n" @@ -165,7 +181,7 @@ def test_block_prod_uses_munderover(self): markdown_text = "$$\\prod_{k=0}^{n} a_k$$" expected = ( - '' + '' "∏k=0" "nak" '\\prod_{k=0}^{n} a_k' @@ -179,7 +195,7 @@ def test_block_int_unaffected(self): markdown_text = "$$\\int_{a}^{b} f(x) dx$$" expected = ( - '' + '' "∫a" 'bf(x)dx' '\\int_{a}^{b} f(x) dx' @@ -202,7 +218,7 @@ def test_simple_inline_math(self): markdown_text = "The variable $$x$$ is unknown." expected = ( "

    The variable " - 'x' + 'x' 'x' " is unknown.

    \n" ) @@ -214,7 +230,7 @@ def test_simple_block_math(self): markdown_text = "$$y = mx + b$$" expected = ( - '' + '' "y=mx+b" 'y = mx + b' "" @@ -227,7 +243,7 @@ def test_mo_accented(self): markdown_text = "$$a_b+\\overrightarrow{abc}+\\overleftarrow{abc}\\div\\surd\\overline{abc}$$" expected = ( - '' + '' "ab+abc→+" 'abc←÷√ab' 'c―' diff --git a/contentcuration/contentcuration/utils/assessment/markdown.py b/contentcuration/contentcuration/utils/assessment/markdown.py index d14a12cd8f..1e0e463a5c 100644 --- a/contentcuration/contentcuration/utils/assessment/markdown.py +++ b/contentcuration/contentcuration/utils/assessment/markdown.py @@ -10,6 +10,7 @@ from markdown_it.utils import EnvType from markdown_it.utils import OptionsDict +from contentcuration.utils.assessment.qti.constants import MATHML_NAMESPACE from contentcuration.utils.assessment.qti.mathml.core import Annotation from contentcuration.utils.assessment.qti.mathml.core import Semantics @@ -120,11 +121,22 @@ def math_block_func( return True -def _convert(latex, inline=True): - # Remove the namespace declaration for cleaner output - markup = convert(latex, display="inline" if inline else "block").replace( - ' xmlns="http://www.w3.org/1998/Math/MathML"', "" - ) +# latex2mathml 3.78.1 writes the argument of \text and its kin into without +# escaping it; the only character reference it emits there is   for a space. +MTEXT_CONTENT_PATTERN = re.compile(r"(]*)?(?)(.*?)()", re.S) +UNESCAPED_AMPERSAND_PATTERN = re.compile(r"&(?!#x[0-9A-Fa-f]+;)") + + +def _escape_mtext(match): + text = UNESCAPED_AMPERSAND_PATTERN.sub("&", match[2]).replace("<", "<") + return f"{match[1]}{text}{match[3]}" + + +def latex_to_mathml(latex, inline=True): + markup = convert(latex, display="inline" if inline else "block") + # ElementTree serializes a parsed default namespace as ns0: prefixes, so parse + # without it and declare it on the root afterwards. + markup = markup.replace(f' xmlns="{MATHML_NAMESPACE}"', "") # By default latex2mathml encodes operators that don't need to be encoded # so we parse it with ElementTree and turn it back into a string here for consistency. math_element = ET.fromstring(markup) @@ -140,6 +152,7 @@ def _convert(latex, inline=True): semantics_element.append(child) semantics_element.append(latex_annotation_element) math_element.append(semantics_element) + math_element.attrib = {"xmlns": MATHML_NAMESPACE, **math_element.attrib} return ET.tostring(math_element, encoding="unicode") @@ -152,7 +165,7 @@ def render_math_inline( env: EnvType, ) -> str: """Render inline math to MathML""" - return _convert(tokens[idx].content) + return latex_to_mathml(tokens[idx].content) def render_math_block( @@ -163,7 +176,7 @@ def render_math_block( env: EnvType, ) -> str: """Render block math to MathML""" - return _convert(tokens[idx].content, inline=False) + return latex_to_mathml(tokens[idx].content, inline=False) def texmath_to_mathml_plugin(md: MarkdownIt) -> None: diff --git a/contentcuration/contentcuration/utils/assessment/qti/base.py b/contentcuration/contentcuration/utils/assessment/qti/base.py index 9196fab4c5..113b7cefe4 100644 --- a/contentcuration/contentcuration/utils/assessment/qti/base.py +++ b/contentcuration/contentcuration/utils/assessment/qti/base.py @@ -18,6 +18,7 @@ from pydantic import PrivateAttr from .constants import Dir +from .constants import MATHML_NAMESPACE from .fields import entry_pattern as srcset_entry_pattern from contentcuration.utils.assessment.qti.fields import BCP47Language @@ -212,7 +213,8 @@ def _ensure_registry_complete(cls): def get_class_for_tag(cls, tag_name: str) -> Optional[Type]: """Get the registered class for a given tag name""" cls._ensure_registry_complete() - return getattr(cls, "_registry", {}).get(tag_name) + local_name = tag_name.removeprefix(f"{{{MATHML_NAMESPACE}}}") + return getattr(cls, "_registry", {}).get(local_name) @lru_cache(maxsize=1) diff --git a/contentcuration/contentcuration/utils/assessment/qti/constants.py b/contentcuration/contentcuration/utils/assessment/qti/constants.py index 05d530734a..27bd391605 100644 --- a/contentcuration/contentcuration/utils/assessment/qti/constants.py +++ b/contentcuration/contentcuration/utils/assessment/qti/constants.py @@ -1,6 +1,8 @@ from enum import Enum +MATHML_NAMESPACE = "http://www.w3.org/1998/Math/MathML" + # QTI Constants for Outcome Declarations diff --git a/contentcuration/contentcuration/utils/assessment/qti/mathml/core.py b/contentcuration/contentcuration/utils/assessment/qti/mathml/core.py index 4bd3540fd0..6f7c53e99d 100644 --- a/contentcuration/contentcuration/utils/assessment/qti/mathml/core.py +++ b/contentcuration/contentcuration/utils/assessment/qti/mathml/core.py @@ -19,13 +19,14 @@ from .fields import MathMLDisplay from .fields import MathMLForm from contentcuration.utils.assessment.qti.base import TextType - +from contentcuration.utils.assessment.qti.constants import MATHML_NAMESPACE PresentationContentListLength2 = Annotated[PresentationContentList, Len(2, 2)] PresentationContentListLength3 = Annotated[PresentationContentList, Len(3, 3)] class Math(MathMLElement): + xmlns: str = MATHML_NAMESPACE display: Optional[MathMLDisplay] = None alttext: Optional[str] = None children: PresentationContentList = Field(default_factory=list) From ac9d745d5702c06b7785f9fa9767e0682b84a3f5 Mon Sep 17 00:00:00 2001 From: rtibblesbot Date: Tue, 27 Oct 2026 15:53:29 -0700 Subject: [PATCH 2/7] Render unconvertible LaTeX as its source instead of raising latex2mathml raises on malformed input such as `x_` or `{`, which aborted the whole channel publish. The formula now renders as escaped `$$latex$$` text and a warning is logged. Co-Authored-By: Claude Opus 5.5 --- .../tests/utils/qti/test_convert.py | 8 ++++ .../tests/utils/test_markdown.py | 28 +++++++++++++ .../utils/assessment/markdown.py | 40 ++++++++++++++----- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/contentcuration/contentcuration/tests/utils/qti/test_convert.py b/contentcuration/contentcuration/tests/utils/qti/test_convert.py index 59fb2bfe06..d102c79600 100644 --- a/contentcuration/contentcuration/tests/utils/qti/test_convert.py +++ b/contentcuration/contentcuration/tests/utils/qti/test_convert.py @@ -586,6 +586,14 @@ def test_maths_in_a_table_cell(self): ) self.assertTrue(validate_qti_item(result.xml.encode("utf-8")).is_valid) + def test_maths_latex2mathml_rejects_converts_to_a_valid_item(self): + for markdown in ("It is $$x_$$ here.", "$${$$", "- $$aIt is $${latex}$$ here.

    \n", + ) + + def test_block_math_latex2mathml_rejects_renders_as_a_paragraph(self): + self.assertEqual( + render_markdown("Before\n\n$$x_$$\n\nAfter"), + "

    Before

    \n

    $$x_$$

    \n

    After

    \n", + ) + + def test_source_of_rejected_math_is_escaped(self): + self.assertEqual( + render_markdown("It is $$aIt is $$a<b&c_$$ here

    \n", + ) + + def test_stray_ampersand_renders_as_its_source(self): + self.assertEqual( + render_markdown("It is $$a & c$$ here"), + "

    It is $$a & c$$ here

    \n", + ) + + class SizedImageTests(unittest.TestCase): """Perseus images, whose size and alignment suffixes are not valid CommonMark.""" diff --git a/contentcuration/contentcuration/utils/assessment/markdown.py b/contentcuration/contentcuration/utils/assessment/markdown.py index 1e0e463a5c..a46269f8ed 100644 --- a/contentcuration/contentcuration/utils/assessment/markdown.py +++ b/contentcuration/contentcuration/utils/assessment/markdown.py @@ -1,8 +1,10 @@ +import logging import re import xml.etree.ElementTree as ET from latex2mathml.converter import convert from markdown_it import MarkdownIt +from markdown_it.common.utils import escapeHtml from markdown_it.renderer import RendererProtocol from markdown_it.rules_block import StateBlock from markdown_it.rules_inline import StateInline @@ -15,6 +17,8 @@ from contentcuration.utils.assessment.qti.mathml.core import Semantics +logger = logging.getLogger(__name__) + # The QTI 3.0 HTML profile has no or , so a decoration is carried as a style # on a . Shared with the reverse conversion in qti/html_to_markdown.py and with # the raw-HTML rewrite in qti/convert.py. @@ -133,13 +137,22 @@ def _escape_mtext(match): def latex_to_mathml(latex, inline=True): - markup = convert(latex, display="inline" if inline else "block") - # ElementTree serializes a parsed default namespace as ns0: prefixes, so parse - # without it and declare it on the root afterwards. - markup = markup.replace(f' xmlns="{MATHML_NAMESPACE}"', "") - # By default latex2mathml encodes operators that don't need to be encoded - # so we parse it with ElementTree and turn it back into a string here for consistency. - math_element = ET.fromstring(markup) + """Return ``None`` when latex2mathml cannot convert ``latex``.""" + try: + markup = convert(latex, display="inline" if inline else "block") + markup = MTEXT_CONTENT_PATTERN.sub(_escape_mtext, markup) + # ElementTree serializes a parsed default namespace as ns0: prefixes, so parse + # without it and declare it on the root afterwards. + markup = markup.replace(f' xmlns="{MATHML_NAMESPACE}"', "") + # By default latex2mathml encodes operators that don't need to be encoded + # so we parse it with ElementTree and turn it back into a string here for consistency. + math_element = ET.fromstring(markup) + # latex2mathml 3.78.1 raises a dozen exception classes with no shared base on + # malformed input, as well as StopIteration, ValueError and IndexError, and emits + # a stray & unescaped, which fails the parse. + except Exception: + logger.warning("Could not convert LaTeX to MathML: %r", latex) + return None # Create LaTeX annotation latex_annotation_element = Annotation( @@ -157,6 +170,10 @@ def latex_to_mathml(latex, inline=True): return ET.tostring(math_element, encoding="unicode") +def latex_source(latex): + return f"$${latex}$$" + + def render_math_inline( self: RendererProtocol, tokens: list[Token], @@ -165,7 +182,8 @@ def render_math_inline( env: EnvType, ) -> str: """Render inline math to MathML""" - return latex_to_mathml(tokens[idx].content) + latex = tokens[idx].content + return latex_to_mathml(latex) or escapeHtml(latex_source(latex)) def render_math_block( @@ -176,7 +194,11 @@ def render_math_block( env: EnvType, ) -> str: """Render block math to MathML""" - return latex_to_mathml(tokens[idx].content, inline=False) + latex = tokens[idx].content + return ( + latex_to_mathml(latex, inline=False) + or f"

    {escapeHtml(latex_source(latex))}

    \n" + ) def texmath_to_mathml_plugin(md: MarkdownIt) -> None: From 808382210dde82e06b2f2ae463a9fe694148a1b9 Mon Sep 17 00:00:00 2001 From: rtibblesbot Date: Tue, 27 Oct 2026 15:53:42 -0700 Subject: [PATCH 3/7] Publish QTI editor formulas as MathML The native QTI publish replaces each `span[data-latex]` the editor wrote with latex_to_mathml output, in prompts, choices and hints. Stored raw_data is left as the editor wrote it. Co-Authored-By: Claude Opus 5.5 --- .../tests/utils/qti/test_media.py | 61 +++++++++++++++++++ .../tests/utils/qti/test_validation.py | 7 +++ .../tests/utils/test_exercise_creation.py | 33 ++++++++-- .../utils/assessment/markdown.py | 3 + .../utils/assessment/qti/archive.py | 2 + .../utils/assessment/qti/media.py | 23 +++++++ 6 files changed, 123 insertions(+), 6 deletions(-) diff --git a/contentcuration/contentcuration/tests/utils/qti/test_media.py b/contentcuration/contentcuration/tests/utils/qti/test_media.py index 9043b263cb..d015f84f8d 100644 --- a/contentcuration/contentcuration/tests/utils/qti/test_media.py +++ b/contentcuration/contentcuration/tests/utils/qti/test_media.py @@ -1,5 +1,11 @@ +from lxml import etree + +from contentcuration.tests.utils.qti.test_validation import SINGLE_HINT_CATALOG from contentcuration.tests.utils.qti.test_validation import VALID_CHOICE_ITEM +from contentcuration.utils.assessment.markdown import latex_to_mathml +from contentcuration.utils.assessment.qti.constants import MATHML_NAMESPACE from contentcuration.utils.assessment.qti.media import get_qti_media_references +from contentcuration.utils.assessment.qti.media import render_qti_latex_spans from contentcuration.utils.assessment.qti.media import rewrite_qti_media_paths from contentcuration.utils.assessment.qti.media import rewrite_qti_sized_image_paths from contentcuration.utils.assessment.qti.media import set_qti_item_language @@ -289,3 +295,58 @@ def test_strip_studio_attributes_ignores_lookalikes_in_values_and_text(): "data-other": "1", } assert paragraph.text == 'data-studio-prompt=""' + + +def _with_latex_spans(prompt, hint): + return VALID_CHOICE_ITEM.replace( + "Select the correct answer.", + f"{prompt}", + ).replace( + "    ', + "Hint.", + ) + ) + annotation = _root(result).find(f".//{{{MATHML_NAMESPACE}}}annotation") + assert annotation.text == r"\begin{matrix}a < b & c\end{matrix}" + validation = validate_qti_item(result) + assert validation.is_valid, validation.errors + + +def test_render_latex_spans_leaves_malformed_latex_as_its_source_text(): + result = render_qti_latex_spans( + _with_latex_spans('Solve now.', "Hint.") + ) + prompt = _root(result).find(".//{*}qti-prompt") + assert list(prompt) == [] + assert prompt.text == "Solve $$x_$$ now." + validation = validate_qti_item(result) + assert validation.is_valid, validation.errors + + +def test_render_latex_spans_leaves_an_item_without_formulas_unchanged(): + assert render_qti_latex_spans(VALID_CHOICE_ITEM) == VALID_CHOICE_ITEM diff --git a/contentcuration/contentcuration/tests/utils/qti/test_validation.py b/contentcuration/contentcuration/tests/utils/qti/test_validation.py index f13d158807..0737ced441 100644 --- a/contentcuration/contentcuration/tests/utils/qti/test_validation.py +++ b/contentcuration/contentcuration/tests/utils/qti/test_validation.py @@ -47,6 +47,13 @@ def _item_xml(identifier, title, response_declaration, item_body): "", ) +SINGLE_HINT_CATALOG = ( + '' + '' + "

    {hint}

    " + "    " +) + _XML_DECLARATION = '' _ENTITY_DOCTYPE = ']>' diff --git a/contentcuration/contentcuration/tests/utils/test_exercise_creation.py b/contentcuration/contentcuration/tests/utils/test_exercise_creation.py index 071bb1545e..fbc50abd91 100644 --- a/contentcuration/contentcuration/tests/utils/test_exercise_creation.py +++ b/contentcuration/contentcuration/tests/utils/test_exercise_creation.py @@ -33,15 +33,18 @@ from contentcuration.tests.testdata import fileobj_exercise_graphie from contentcuration.tests.testdata import fileobj_exercise_image from contentcuration.tests.utils.qti.test_convert import _normalize_xml +from contentcuration.tests.utils.qti.test_media import _with_latex_spans from contentcuration.tests.utils.qti.test_perseus_derive import _text_item from contentcuration.tests.utils.qti.test_perseus_derive import TOLERANCE_ITEM from contentcuration.tests.utils.qti.test_validation import _item_xml +from contentcuration.tests.utils.qti.test_validation import SINGLE_HINT_CATALOG from contentcuration.tests.utils.qti.test_validation import VALID_CHOICE_ITEM from contentcuration.utils.assessment.base import resize_image from contentcuration.utils.assessment.perseus import answerless_input_ids from contentcuration.utils.assessment.perseus import PerseusExerciseGenerator from contentcuration.utils.assessment.qti.archive import hex_to_qti_id from contentcuration.utils.assessment.qti.archive import QTIExerciseGenerator +from contentcuration.utils.assessment.qti.constants import MATHML_NAMESPACE from contentcuration.utils.assessment.qti.validation import parse_qti_xml from contentcuration.utils.assessment.qti.validation import validate_qti_item @@ -147,6 +150,12 @@ def _mpo_bytes(): ) +_FORMULA = '' +FORMULA_CHOICE_ITEM = _with_latex_spans( + f"Is {_FORMULA} even?", f"Try {_FORMULA}." +).replace(">Option A<", f">{_FORMULA}<") + + SVG_BYTES = b'' @@ -2847,6 +2856,23 @@ def test_native_qti_item_published_without_studio_attributes(self): _canonical_xml(raw_data.replace(' data-studio-prompt=""', "")), ) + def test_native_qti_item_formulas_published_as_mathml(self): + item = self._create_native_qti_item(FORMULA_CHOICE_ITEM) + + self._create_qti_zip(_exercise_data([item])) + + exercise_file = self.exercise_node.files.get(preset_id=format_presets.QTI_ZIP) + zip_file = self._validate_qti_zip_structure(exercise_file) + item_xml = zip_file.read("items/item_1.xml") + root = parse_qti_xml(item_xml).getroot() + annotations = root.findall(f".//{{{MATHML_NAMESPACE}}}annotation") + self.assertEqual([a.text for a in annotations], ["x^2"] * 3) + self.assertEqual(root.xpath("//*[@data-latex]"), []) + validation = validate_qti_item(item_xml) + self.assertTrue(validation.is_valid, validation.errors) + item.refresh_from_db() + self.assertEqual(item.raw_data, FORMULA_CHOICE_ITEM) + def test_native_qti_item_media_included_and_addressed(self): # fileobj_exercise_image() writes real bytes to storage keyed by their # actual md5 checksum + "jpg" ext -- use that real checksum/ext rather @@ -3224,12 +3250,7 @@ def test_republish_replaces_stale_native_qti_archive(self): def test_native_qti_perseus_derivation(self): """A native QTI choice item is derived into a rendered Perseus item JSON.""" - catalog_info = ( - '' - '' - "

    First hint.

    " - "    " - ) + catalog_info = SINGLE_HINT_CATALOG.format(hint="First hint.") raw_data = VALID_CHOICE_ITEM.replace( " Optional[Tuple[str, bytes] # depending only on which editor wrote them. item_xml = set_qti_item_language(item_xml, self._node_language()) item_xml = strip_studio_attributes(item_xml) + item_xml = render_qti_latex_spans(item_xml) self._add_resource( QTIResource( diff --git a/contentcuration/contentcuration/utils/assessment/qti/media.py b/contentcuration/contentcuration/utils/assessment/qti/media.py index 9222eb0e2e..3b4392d813 100644 --- a/contentcuration/contentcuration/utils/assessment/qti/media.py +++ b/contentcuration/contentcuration/utils/assessment/qti/media.py @@ -4,6 +4,9 @@ from lxml import etree +from contentcuration.utils.assessment.markdown import LATEX_ATTRIBUTE +from contentcuration.utils.assessment.markdown import latex_source +from contentcuration.utils.assessment.markdown import latex_to_mathml from contentcuration.utils.assessment.qti.fields import ( entry_pattern as srcset_entry_pattern, ) @@ -18,6 +21,8 @@ ) PIXEL_LENGTH_REGEX = re.compile(r"[1-9][0-9]*") XML_LANG_ATTRIBUTE = "{http://www.w3.org/XML/1998/namespace}lang" +LATEX_SPAN_XPATH = etree.XPath(f"//*[local-name()='span'][@{LATEX_ATTRIBUTE}]") +UNCONVERTED_LATEX_TAG = "unconverted-latex" def img_pixel_size(width, height) -> Optional[Tuple[int, int]]: @@ -136,3 +141,21 @@ def strip_studio_attributes(raw_data): if name.startswith("data-studio-"): del element.attrib[name] return _serialize(doc) + + +def render_qti_latex_spans(raw_data): + if LATEX_ATTRIBUTE not in raw_data: + return raw_data + doc = parse_qti_xml(raw_data) + for span in LATEX_SPAN_XPATH(doc): + latex = span.attrib.pop(LATEX_ATTRIBUTE) + mathml = latex_to_mathml(latex) + if mathml is None: + span.text = latex_source(latex) + span.tag = UNCONVERTED_LATEX_TAG + continue + math = etree.fromstring(mathml) + math.tail = span.tail + span.getparent().replace(span, math) + etree.strip_tags(doc, UNCONVERTED_LATEX_TAG) + return _serialize(doc) From 7178d4e8b22177e7e3f2d6e588dfbedd9cd882b9 Mon Sep 17 00:00:00 2001 From: rtibblesbot Date: Tue, 27 Oct 2026 16:17:28 -0700 Subject: [PATCH 4/7] Keep QTI editor formulas in the derived Perseus exercise A `span[data-latex]` used to unwrap to its empty content, so the formula vanished. It now derives as `$$latex$$`. Co-Authored-By: Claude Opus 5.5 --- .../tests/utils/qti/test_html_to_markdown.py | 7 +++++++ .../tests/utils/test_exercise_creation.py | 16 ++++++++++++++++ .../utils/assessment/qti/html_to_markdown.py | 14 ++++++++++---- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/contentcuration/contentcuration/tests/utils/qti/test_html_to_markdown.py b/contentcuration/contentcuration/tests/utils/qti/test_html_to_markdown.py index 0cf4d04783..491b563530 100644 --- a/contentcuration/contentcuration/tests/utils/qti/test_html_to_markdown.py +++ b/contentcuration/contentcuration/tests/utils/qti/test_html_to_markdown.py @@ -97,6 +97,13 @@ def test_underlined_span_becomes_a_perseus_underline(): ) +def test_formula_span_becomes_double_dollar_latex(): + assert ( + _markdown_from_html('

    Is even?

    ') + == "Is $$x^2$$ even?" + ) + + def test_span_carrying_both_decorations_keeps_both(): assert ( _markdown_from_html( diff --git a/contentcuration/contentcuration/tests/utils/test_exercise_creation.py b/contentcuration/contentcuration/tests/utils/test_exercise_creation.py index fbc50abd91..c05bd89255 100644 --- a/contentcuration/contentcuration/tests/utils/test_exercise_creation.py +++ b/contentcuration/contentcuration/tests/utils/test_exercise_creation.py @@ -3302,6 +3302,22 @@ def test_native_qti_perseus_derivation(self): hint_content = "".join(hint["content"] for hint in item_json["hints"]) self.assertIn("First hint.", hint_content) + def test_native_qti_perseus_derivation_keeps_formulas(self): + item = self._create_native_qti_item(FORMULA_CHOICE_ITEM) + + self._create_perseus_zip(_exercise_data([item])) + + exercise_file = self.exercise_node.files.get(preset_id=format_presets.EXERCISE) + with storage.open(exercise_file.file_on_disk.name, "rb") as f: + zip_file = zipfile.ZipFile(BytesIO(f.read())) + item_json = json.loads(zip_file.read("item_1.json").decode("utf-8")) + self.assertIn("Is $x^2$ even?", item_json["question"]["content"]) + choices = item_json["question"]["widgets"]["radio 1"]["options"]["choices"] + self.assertIn("$x^2$", [choice["content"] for choice in choices]) + self.assertEqual( + [hint["content"] for hint in item_json["hints"]], ["Try $x^2$."] + ) + def test_native_qti_perseus_derivation_resizes_sized_images(self): image_file = fileobj_exercise_image(size=(400, 300), color="blue") filename = image_file.filename() diff --git a/contentcuration/contentcuration/utils/assessment/qti/html_to_markdown.py b/contentcuration/contentcuration/utils/assessment/qti/html_to_markdown.py index 25af3f3c87..1975919b10 100644 --- a/contentcuration/contentcuration/utils/assessment/qti/html_to_markdown.py +++ b/contentcuration/contentcuration/utils/assessment/qti/html_to_markdown.py @@ -25,6 +25,8 @@ from le_utils.constants import exercises from lxml import etree +from contentcuration.utils.assessment.markdown import LATEX_ATTRIBUTE +from contentcuration.utils.assessment.markdown import latex_source from contentcuration.utils.assessment.markdown import STRIKETHROUGH_DECORATION from contentcuration.utils.assessment.markdown import UNDERLINE_DECORATION from contentcuration.utils.assessment.qti.media import img_pixel_size @@ -67,7 +69,7 @@ def _render_math(el): _localname(annotation) == "annotation" and annotation.get("encoding") == "application/x-tex" ): - return "$${}$${}".format(annotation.text or "", suffix) + return latex_source(annotation.text or "") + suffix logger.warning("MathML element without an application/x-tex annotation; dropping") return "" @@ -102,10 +104,14 @@ def _render_span(el): """A decorated span is how a strikethrough and an underline are carried. ``render_markdown`` writes the first; the editor writes either, since the QTI - HTML profile has no ```` and no ````. Any other span unwraps, keeping its - content: whatever a foreign QTI package styles a span with has nothing in - Perseus markdown to say it. + HTML profile has no ```` and no ````. The editor also writes a formula as + an empty span carrying its LaTeX. Any other span unwraps, keeping its content: + whatever a foreign QTI package styles a span with has nothing in Perseus + markdown to say it. """ + latex = el.get(LATEX_ATTRIBUTE) + if latex is not None: + return latex_source(latex) decorations = _text_decorations(el) rendered = _render_inline(el) if UNDERLINE_DECORATION in decorations: From ed792aff0c414d1700cd62a9a74fe722ac29ec5c Mon Sep 17 00:00:00 2001 From: rtibblesbot Date: Tue, 27 Oct 2026 16:17:29 -0700 Subject: [PATCH 5/7] Test that restored editor formulas republish unchanged Restore keeps the packaged MathML, which the editor loads as formulas. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_restore_channel.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/contentcuration/contentcuration/tests/test_restore_channel.py b/contentcuration/contentcuration/tests/test_restore_channel.py index 6e2010d170..8d81c036a8 100644 --- a/contentcuration/contentcuration/tests/test_restore_channel.py +++ b/contentcuration/contentcuration/tests/test_restore_channel.py @@ -447,6 +447,22 @@ def test_qti_only_node_restores_editor_and_legacy_items_as_qti(self): self.assertIn("Legacy question", items[1].raw_data) self._assert_republishes_source_package(QTIExerciseGenerator) + def test_editor_item_formula_restores_as_published_mathml(self): + item = self._add_editor_item(0) + item.raw_data = item.raw_data.replace( + "Pick one.", 'Pick one.' + ) + item.save() + self._publish(QTIExerciseGenerator) + + items = self._restore(format_presets.QTI_ZIP) + + self.assertIn( + 'x^2', + items[0].raw_data, + ) + self._assert_republishes_source_package(QTIExerciseGenerator) + def test_editor_item_with_non_canonical_id_gets_a_new_assessment_id(self): # Matches hex_to_qti_id's shape but sets its padding bits. identifier = "K" + "A" * 21 + "B" From 77d7c38da942f0f8dee31c3fe1f525fe19ff1cf0 Mon Sep 17 00:00:00 2001 From: rtibblesbot Date: Tue, 27 Oct 2026 16:47:53 -0700 Subject: [PATCH 6/7] Keep text after a formula that starts a legacy line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A line opening with $$…$$ was parsed as block math, dropping the rest of the line (`$$5$$ exactly` lost "exactly"). Block math now needs the formula to end its line. Co-Authored-By: Claude Opus 5.5 --- .../tests/utils/test_markdown.py | 25 +++++++++++++++++++ .../utils/assessment/markdown.py | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/contentcuration/contentcuration/tests/utils/test_markdown.py b/contentcuration/contentcuration/tests/utils/test_markdown.py index 41be8c91cc..322c4fc771 100644 --- a/contentcuration/contentcuration/tests/utils/test_markdown.py +++ b/contentcuration/contentcuration/tests/utils/test_markdown.py @@ -238,6 +238,31 @@ def test_simple_block_math(self): self._assert_conversion(markdown_text, expected) + def test_math_at_line_start_followed_by_text_is_inline(self): + markdown_text = "$$5$$ exactly" + expected = ( + "

    " + '5' + '5' + " exactly

    \n" + ) + + self._assert_conversion(markdown_text, expected) + + def test_line_starting_and_ending_with_math_is_inline(self): + markdown_text = "$$a$$ and $$b$$" + expected = ( + "

    " + 'a' + 'a' + " and " + 'b' + 'b' + "

    \n" + ) + + self._assert_conversion(markdown_text, expected) + def test_mo_accented(self): """Regression test for missed experimental property on mo tags""" diff --git a/contentcuration/contentcuration/utils/assessment/markdown.py b/contentcuration/contentcuration/utils/assessment/markdown.py index 0469e1be0e..0a5e0fdc49 100644 --- a/contentcuration/contentcuration/utils/assessment/markdown.py +++ b/contentcuration/contentcuration/utils/assessment/markdown.py @@ -30,7 +30,7 @@ # Regex patterns for $$ delimited math INLINE_PATTERN = re.compile(r"^\$\$([\s\S]+?)\$\$") -BLOCK_PATTERN = re.compile(r"^\$\$([\s\S]+?)\$\$", re.M) +BLOCK_PATTERN = re.compile(r"^\$\$((?:(?!\$\$)[\s\S])+)\$\$[ \t]*$", re.M) # Perseus extends the CommonMark image with a trailing size and alignment: # ![alt](. =550x364 align=center) From 86f1e956c04d3eb548e862723e1a84c37d59ee52 Mon Sep 17 00:00:00 2001 From: rtibblesbot Date: Tue, 27 Oct 2026 16:47:54 -0700 Subject: [PATCH 7/7] Load annotated MathML as editor formulas Converted legacy items carry formulas as MathML, which TipTap flattened to text, MathML and LaTeX run together, so saving an edit destroyed them. The Math node now parses MathML with a TeX annotation. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/formulaItem.spec.js | 55 +++++++++++++++++++ .../TipTapEditor/extensions/Math.js | 19 ++++++- .../TipTapEditor/utils/comparisonKey.js | 4 +- .../__tests__/comparisonKey.spec.js | 3 + 4 files changed, 79 insertions(+), 2 deletions(-) diff --git a/contentcuration/contentcuration/frontend/shared/views/QTIEditor/serialization/__tests__/formulaItem.spec.js b/contentcuration/contentcuration/frontend/shared/views/QTIEditor/serialization/__tests__/formulaItem.spec.js index e1ce6a7044..a8602981e4 100644 --- a/contentcuration/contentcuration/frontend/shared/views/QTIEditor/serialization/__tests__/formulaItem.spec.js +++ b/contentcuration/contentcuration/frontend/shared/views/QTIEditor/serialization/__tests__/formulaItem.spec.js @@ -115,3 +115,58 @@ it('keeps the text after a formula in a hint once loaded into the editor', () => const item = buildItem(INTERACTIONS[0]); expect(load(item.hints[0].content)).toBe(CONTENT); }); + +describe('an item converted from legacy markdown with formulas', () => { + const inlineMath = (latex, mathml) => + '' + + `${mathml}${latex}` + + ''; + const blockMath = (latex, mathml) => + inlineMath(latex, mathml).replace('display="inline"', 'display="block"'); + const RAW_DATA = + '' + + 'choice_0' + + '' + + '' + + `

    What is ${inlineMath('x^2 < 1', 'x')} when x is 2?

    ` + + blockMath('\\frac{a}{b}', 'a') + + '    ' + + blockMath('\\frac{1}{2}', '1') + + '' + + '

    Other

    ' + + '' + + `

    Square it: ${inlineMath('2^2', '2')} first

    ` + + '    '; + + let item; + let state; + + beforeAll(() => { + item = parseItem(RAW_DATA); + const [interaction] = item.interactions; + state = choiceInteractionDescriptor.parse( + interaction.bodyXml, + interaction.responseDeclarations, + ); + }); + + it('loads inline and block formulas in the prompt as editor formulas', () => { + expect(load(state.prompt)).toBe( + '

    What is when x is 2?

    ' + + '

    ', + ); + }); + + it('loads a formula answer as an editor formula', () => { + expect(load(state.choices[0].content)).toBe('

    '); + }); + + it('loads a formula in a hint as an editor formula', () => { + expect(load(item.hints[0].content)).toBe( + '

    Square it: first

    ', + ); + }); +}); diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Math.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Math.js index 5469073162..a62e4eb503 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Math.js +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Math.js @@ -2,6 +2,14 @@ import { Node, mergeAttributes } from '@tiptap/core'; import { VueNodeViewRenderer } from '@tiptap/vue-2'; import LazyMathNodeView from '../components/math/LazyMathNodeView.vue'; +/** + * @param {Element} math - A MathML `` element + * @returns {string|undefined} The LaTeX it was rendered from, as published formulas annotate it + */ +export function mathmlLatex(math) { + return math.querySelector('annotation[encoding="application/x-tex"]')?.textContent; +} + export const Math = Node.create({ name: 'math', group: 'inline', @@ -23,7 +31,16 @@ export const Math = Node.create({ }, parseHTML() { - return [{ tag: 'span[data-latex]' }]; + return [ + { tag: 'span[data-latex]' }, + { + tag: 'math', + getAttrs: element => { + const latex = mathmlLatex(element); + return latex ? { latex } : false; + }, + }, + ]; }, renderHTML({ HTMLAttributes }) { diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/comparisonKey.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/comparisonKey.js index f04d0c9f02..ce09adb6de 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/comparisonKey.js +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/comparisonKey.js @@ -4,6 +4,7 @@ import { getSchema } from '@tiptap/core'; // eslint-disable-next-line import/no-unresolved import { DOMParser as PMDOMParser } from '@tiptap/pm/model'; import { editorExtensions } from '../extensions'; +import { mathmlLatex } from '../extensions/Math'; // The full schema, so inline and block content of the same text read the same. const schema = getSchema(editorExtensions()); @@ -60,7 +61,7 @@ export function hasRichTextContent(content) { * * Its text, however marked up except its sub/superscripts, and each formula, image, * SVG, MathML and line break where it stands. Read as TipTap saves it, so its - * re-render keys the same, unless it has SVG or MathML, which TipTap drops. + * re-render keys the same, unless it has SVG or MathML without its LaTeX, which TipTap drops. * * @param {string} html - HTML fragment from a TipTap editor * @returns {string} '' for a fragment without rich text content @@ -72,6 +73,7 @@ export function richTextComparisonKey(html) { // into U+FFFD, so no stored formula has this LaTeX. const unparsed = new Map(); for (const element of document.body.querySelectorAll(UNPARSED_MEDIA)) { + if (element.localName === 'math' && mathmlLatex(element)) continue; const latex = `\0${unparsed.size}`; unparsed.set(latex, ['html', element.outerHTML.replace(/\s+/g, ' ').replace(/>\s+<')]); const placeholder = document.createElement('span'); diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/__tests__/comparisonKey.spec.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/__tests__/comparisonKey.spec.js index 119be0ba99..983d3dba35 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/__tests__/comparisonKey.spec.js +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/__tests__/comparisonKey.spec.js @@ -205,6 +205,9 @@ describe('richTextComparisonKey', () => { '
    if x:\n    return 1
    ', '

    a
    b

    ', '

    a
    \n b

    ', + '

    Solve ' + + 'xx' + + ' now

    ', ])('%j keys the same', stored => { expect(richTextComparisonKey(rewrap(stored))).toBe(richTextComparisonKey(stored)); });