From 8767cdce02dd4f8c2fa4167ad4e3c57f2a4db366 Mon Sep 17 00:00:00 2001 From: rtibblesbot Date: Fri, 30 Oct 2026 17:11:24 -0700 Subject: [PATCH 1/2] Add mappings namedtuple spec option Co-Authored-By: Claude Opus 5.5 --- scripts/generate_from_specs.py | 48 ++++++++++++++++++++++++++++++- tests/test_generate_from_specs.py | 45 +++++++++++++++++++++++++---- 2 files changed, 87 insertions(+), 6 deletions(-) diff --git a/scripts/generate_from_specs.py b/scripts/generate_from_specs.py index eb659ab..89e2337 100644 --- a/scripts/generate_from_specs.py +++ b/scripts/generate_from_specs.py @@ -274,6 +274,29 @@ def namedtuple_tuples(spec, constants): return tuples +def namedtuple_mappings(spec, constants): + mappings = OrderedDict() + name = spec["namedtuple"]["name"] + defined_names = set(constants) | set(spec.get("tuples", {})) + defined_names |= {name, "{}LIST".format(name.upper()), "choices", "default", "generate_list", "namedtuple", "unicode_literals"} + if "getter" in spec["namedtuple"]: + defined_names |= {spec["namedtuple"]["getter"], "_{}LOOKUP".format(name.upper())} + for mapping_name, entries in spec.get("mappings", {}).items(): + if mapping_name in defined_names: + raise ValueError("Mapping {} is already defined".format(mapping_name)) + mapping = mappings[mapping_name] = OrderedDict() + for constant, keys in entries.items(): + if constant not in constants: + raise ValueError("Mapping {} references undefined constant {}".format(mapping_name, constant)) + if not isinstance(keys, list): + raise ValueError("Mapping {} keys for {} are not a list".format(mapping_name, constant)) + for key in keys: + if key in mapping: + raise ValueError("Mapping {} maps {} twice".format(mapping_name, key)) + mapping[key] = constant + return mappings + + def namedtuple_choices(spec): if "choices_order" not in spec: return None @@ -315,6 +338,15 @@ def write_python_records(f, spec): f.write("]\n") +def write_python_mappings(f, mappings): + for mapping_name, mapping in mappings.items(): + f.write("\n") + f.write("{} = {{\n".format(mapping_name)) + for key, constant in mapping.items(): + f.write(" {!r}: {},\n".format(key, constant)) + f.write("}\n") + + def write_python_getter(f, spec): upper = spec["namedtuple"]["name"].upper() f.write("\n") @@ -328,6 +360,7 @@ def write_python_getter(f, spec): def write_namedtuple_python_file(output_file, name, spec): constants = namedtuple_constants(spec) tuples = namedtuple_tuples(spec, constants) + mappings = namedtuple_mappings(spec, constants) choices = namedtuple_choices(spec) with open(output_file, "w") as f: write_python_header(f) @@ -340,6 +373,7 @@ def write_namedtuple_python_file(output_file, name, spec): for tuple_name, members in tuples.items(): f.write("\n") f.write("{} = ({},)\n".format(tuple_name, ", ".join(members))) + write_python_mappings(f, mappings) f.write("\n") if choices is not None: write_python_choices(f, choices) @@ -354,14 +388,25 @@ def js_object(pairs): return "{{ {} }}".format(", ".join("{}: {}".format(key, json.dumps(value)) for key, value in pairs)) +def write_js_mappings(f, mappings, constants): + for mapping_name, mapping in mappings.items(): + f.write("\n") + f.write("export const {} = {{\n".format(mapping_name)) + for key, constant in mapping.items(): + f.write(" {}: {},\n".format(json.dumps(key), json.dumps(constants[constant]))) + f.write("};\n") + + def write_namedtuple_js_file(output_file, name, spec): fields = spec["namedtuple"]["fields"] + constants = namedtuple_constants(spec) + mappings = namedtuple_mappings(spec, constants) with open(output_file, "w") as f: write_js_header(f) f.write("// {}\n".format(name)) f.write("\n") f.write("export default {\n") - for constant, value in namedtuple_constants(spec).items(): + for constant, value in constants.items(): f.write(" {}: {},\n".format(constant, json.dumps(value))) f.write("};\n") f.write("\n") @@ -369,6 +414,7 @@ def write_namedtuple_js_file(output_file, name, spec): for record in spec["records"]: f.write(" {},\n".format(js_object((field, record[field]) for field in fields))) f.write("];\n") + write_js_mappings(f, mappings, constants) def write_labels_src_files(label_outputs): diff --git a/tests/test_generate_from_specs.py b/tests/test_generate_from_specs.py index 2a5b635..950f990 100644 --- a/tests/test_generate_from_specs.py +++ b/tests/test_generate_from_specs.py @@ -90,6 +90,26 @@ def test_tuple_repeating_constant_raises(tmp_path): assert_spec_raises(tmp_path, spec(tuples={"VIDEO": ["MP4", "MP4"]})) +def test_mapping_with_undefined_constant_raises(tmp_path): + assert_spec_raises(tmp_path, spec(mappings={"MAPPING": {"AVI": ["avi"]}})) + + +def test_mapping_key_under_two_constants_raises(tmp_path): + mappings = {"MAPPING": {"MP4": ["a"], "AVI": ["a"]}} + assert_spec_raises(tmp_path, spec(constants={"AVI": "avi"}, mappings=mappings)) + + +def test_mapping_name_already_defined_raises(tmp_path): + (tmp_path / "valid").mkdir() + names = set(write_and_exec(tmp_path / "valid", spec(tuples={"VIDEO": ["MP4"]}))) - {"__builtins__"} + for mapping_name in names | {"default"}: + assert_spec_raises(tmp_path, spec(tuples={"VIDEO": ["MP4"]}, mappings={mapping_name: {"MP4": ["a"]}})) + + +def test_mapping_keys_not_a_list_raises(tmp_path): + assert_spec_raises(tmp_path, spec(mappings={"MAPPING": {"MP4": "mp4"}})) + + def test_record_missing_field_raises(tmp_path): record = mp4_record() del record["mimetype"] @@ -126,6 +146,11 @@ def test_tuple_with_defined_constants_is_written(tmp_path): assert namespace["VIDEO"] == ("mp4", "avi") +def test_mapping_maps_keys_to_constant_values(tmp_path): + namespace = write_and_exec(tmp_path, spec(constants={"AVI": "avi"}, mappings={"MAPPING": {"MP4": ["a", "b"], "AVI": ["c"]}})) + assert namespace["MAPPING"] == {"a": "mp4", "b": "mp4", "c": "avi"} + + def test_generate_list_yields_records(tmp_path): namespace = write_and_exec(tmp_path, spec()) records = list(namespace["generate_list"]({"webm": {"mimetype": "video/webm"}})) @@ -152,12 +177,22 @@ def test_constant_field_sets_record_constant_value(tmp_path): assert namespace["MP4"] == "video/mp4" -@pytest.mark.skipif(shutil.which("node") is None, reason="node is not installed") -def test_js_exports_constants_and_records(tmp_path): +def import_js(tmp_path, valid_spec, imports, value): output_file = tmp_path / "Formats.mjs" - generate_from_specs.write_namedtuple_js_file(str(output_file), "FileFormats", spec(constants={"AVI": "avi"})) - script = "import constants, {{ FORMATLIST }} from {!r}; console.log(JSON.stringify([constants, FORMATLIST]));".format(output_file.as_uri()) + generate_from_specs.write_namedtuple_js_file(str(output_file), "FileFormats", valid_spec) + script = "import {} from {!r}; console.log(JSON.stringify({}));".format(imports, output_file.as_uri(), value) output = subprocess.check_output(["node", "--input-type=module", "-e", script]) - constants, records = json.loads(output.decode("utf-8"), object_pairs_hook=OrderedDict) + return json.loads(output.decode("utf-8"), object_pairs_hook=OrderedDict) + + +@pytest.mark.skipif(shutil.which("node") is None, reason="node is not installed") +def test_js_exports_constants_and_records(tmp_path): + constants, records = import_js(tmp_path, spec(constants={"AVI": "avi"}), "constants, { FORMATLIST }", "[constants, FORMATLIST]") assert list(constants.items()) == [("MP4", "mp4"), ("MP4_MIMETYPE", "video/mp4"), ("AVI", "avi")] assert records == [{"id": "mp4", "mimetype": "video/mp4"}] + + +@pytest.mark.skipif(shutil.which("node") is None, reason="node is not installed") +def test_js_exports_mappings(tmp_path): + valid_spec = spec(constants={"AVI": "avi"}, mappings={"MAPPING": {"MP4": ["a", "b"], "AVI": ["c-d"]}}) + assert import_js(tmp_path, valid_spec, "{ MAPPING }", "MAPPING") == {"a": "mp4", "b": "mp4", "c-d": "avi"} From 97310d478bbd648893ae3e0bbc6dd1aed60164d2 Mon Sep 17 00:00:00 2001 From: rtibblesbot Date: Fri, 30 Oct 2026 17:11:24 -0700 Subject: [PATCH 2/2] Generate content_kinds from a namedtuple spec Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 + README.md | 2 +- js/ContentKinds.js | 43 ++++++++++++++ le_utils/constants/content_kinds.py | 64 ++++++++++---------- le_utils/resources/kindlookup.json | 32 ---------- spec/constants-content_kinds.json | 91 +++++++++++++++++++++++++++++ tests/test_kinds.py | 18 +++--- 7 files changed, 178 insertions(+), 74 deletions(-) create mode 100644 js/ContentKinds.js delete mode 100644 le_utils/resources/kindlookup.json create mode 100644 spec/constants-content_kinds.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cd2b9b..eb29145 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ Unreleased - Added `kolibri-constants/Licenses` - `licenses.generate_list` no longer casts ids to `int` or fills `description` - `archive.contents_sha256` reads members in stored order + - Removed `resources/kindlookup.json`; use `content_kinds.KINDLIST` + - Added `kolibri-constants/ContentKinds` 0.2.17 Apr 13, 2026 diff --git a/README.md b/README.md index 5494d61..cb0d831 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ Content kinds (ContentNode subclasses) Content items throughout the Kolibri ecosystem come in several kinds. The `kind` attribute of each object can be one of ("topic", "video", "audio", "exercise" "document", or "html5". -See [constants/content_kinds.py](https://github.com/learningequality/le-utils/blob/master/le_utils/constants/content_kinds.py#L11-L17) for latest list. +See [constants/content_kinds.py](https://github.com/learningequality/le-utils/blob/master/le_utils/constants/content_kinds.py#L9-L18) for latest list. The currently supported content kinds are: - Topic node (folder) diff --git a/js/ContentKinds.js b/js/ContentKinds.js new file mode 100644 index 0000000..726394a --- /dev/null +++ b/js/ContentKinds.js @@ -0,0 +1,43 @@ +// -*- coding: utf-8 -*- +// Generated by scripts/generate_from_specs.py +// ContentKinds + +export default { + TOPIC: "topic", + VIDEO: "video", + AUDIO: "audio", + EXERCISE: "exercise", + DOCUMENT: "document", + HTML5: "html5", + SLIDESHOW: "slideshow", + H5P: "h5p", + ZIM: "zim", + QUIZ: "quiz", +}; + +export const KINDLIST = [ + { id: "topic", name: "topic" }, + { id: "video", name: "video" }, + { id: "audio", name: "audio" }, + { id: "exercise", name: "exercise" }, + { id: "document", name: "document" }, + { id: "html5", name: "html5" }, + { id: "slideshow", name: "slideshow" }, + { id: "h5p", name: "h5p" }, + { id: "zim", name: "zim" }, + { id: "quiz", name: "quiz" }, +]; + +export const MAPPING = { + "mp4": "video", + "webm": "video", + "mp3": "audio", + "perseus": "exercise", + "pdf": "document", + "epub": "document", + "bloompub": "document", + "bloomd": "document", + "zip": "html5", + "h5p": "h5p", + "zim": "zim", +}; diff --git a/le_utils/constants/content_kinds.py b/le_utils/constants/content_kinds.py index b2896c2..b0ba617 100644 --- a/le_utils/constants/content_kinds.py +++ b/le_utils/constants/content_kinds.py @@ -1,12 +1,10 @@ -import json -import pkgutil -from collections import namedtuple - -from le_utils.constants import file_formats +# -*- coding: utf-8 -*- +# Generated by scripts/generate_from_specs.py +from __future__ import unicode_literals -""" Content Kind Constants """ +from collections import namedtuple -# IMPORTANT: Keep these constants in sync with kindlookup.json and presetlookup.json +# ContentKinds TOPIC = "topic" VIDEO = "video" @@ -19,6 +17,20 @@ ZIM = "zim" QUIZ = "quiz" +MAPPING = { + "mp4": VIDEO, + "webm": VIDEO, + "mp3": AUDIO, + "perseus": EXERCISE, + "pdf": DOCUMENT, + "epub": DOCUMENT, + "bloompub": DOCUMENT, + "bloomd": DOCUMENT, + "zip": HTML5, + "h5p": H5P, + "zim": ZIM, +} + choices = ( (TOPIC, "Topic"), (VIDEO, "Video"), @@ -32,35 +44,25 @@ (QUIZ, "Quiz"), ) -""" File Format (extension) to Content Kind Mapping """ -MAPPING = { - file_formats.MP4: VIDEO, - file_formats.WEBM: VIDEO, - file_formats.MP3: AUDIO, - file_formats.PDF: DOCUMENT, - file_formats.EPUB: DOCUMENT, - file_formats.PERSEUS: EXERCISE, - file_formats.HTML5: HTML5, - file_formats.H5P: H5P, - file_formats.ZIM: ZIM, - file_formats.BLOOMPUB: DOCUMENT, - file_formats.BLOOMD: DOCUMENT, -} - class Kind(namedtuple("Kind", ["id", "name"])): pass def generate_list(constantlist): - for id, kind in constantlist.items(): - yield Kind(id=id, **kind) - - -def _initialize_kind_list(): - constantlist = json.loads(pkgutil.get_data("le_utils", "resources/kindlookup.json").decode("utf-8")) - - return generate_list(constantlist) + for id, record in constantlist.items(): + yield Kind(id=id, **record) -KINDLIST = list(_initialize_kind_list()) +KINDLIST = [ + Kind(id="topic", name="topic"), + Kind(id="video", name="video"), + Kind(id="audio", name="audio"), + Kind(id="exercise", name="exercise"), + Kind(id="document", name="document"), + Kind(id="html5", name="html5"), + Kind(id="slideshow", name="slideshow"), + Kind(id="h5p", name="h5p"), + Kind(id="zim", name="zim"), + Kind(id="quiz", name="quiz"), +] diff --git a/le_utils/resources/kindlookup.json b/le_utils/resources/kindlookup.json deleted file mode 100644 index ed9a19d..0000000 --- a/le_utils/resources/kindlookup.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "topic": { - "name":"topic" - }, - "video": { - "name":"video" - }, - "audio": { - "name":"audio" - }, - "exercise": { - "name":"exercise" - }, - "document": { - "name":"document" - }, - "html5": { - "name":"html5" - }, - "slideshow": { - "name": "slideshow" - }, - "h5p": { - "name": "h5p" - }, - "zim": { - "name": "zim" - }, - "quiz": { - "name": "quiz" - } -} diff --git a/spec/constants-content_kinds.json b/spec/constants-content_kinds.json new file mode 100644 index 0000000..bb521ec --- /dev/null +++ b/spec/constants-content_kinds.json @@ -0,0 +1,91 @@ +{ + "namedtuple": { + "name": "Kind", + "fields": ["id", "name"] + }, + "records": [ + { + "constant": "TOPIC", + "label": "Topic", + "id": "topic", + "name": "topic" + }, + { + "constant": "VIDEO", + "label": "Video", + "id": "video", + "name": "video" + }, + { + "constant": "AUDIO", + "label": "Audio", + "id": "audio", + "name": "audio" + }, + { + "constant": "EXERCISE", + "label": "Exercise", + "id": "exercise", + "name": "exercise" + }, + { + "constant": "DOCUMENT", + "label": "Document", + "id": "document", + "name": "document" + }, + { + "constant": "HTML5", + "label": "HTML5 App", + "id": "html5", + "name": "html5" + }, + { + "constant": "SLIDESHOW", + "label": "Slideshow", + "id": "slideshow", + "name": "slideshow" + }, + { + "constant": "H5P", + "label": "H5P", + "id": "h5p", + "name": "h5p" + }, + { + "constant": "ZIM", + "label": "Zim", + "id": "zim", + "name": "zim" + }, + { + "constant": "QUIZ", + "label": "Quiz", + "id": "quiz", + "name": "quiz" + } + ], + "choices_order": [ + "TOPIC", + "VIDEO", + "AUDIO", + "EXERCISE", + "DOCUMENT", + "HTML5", + "SLIDESHOW", + "H5P", + "ZIM", + "QUIZ" + ], + "mappings": { + "MAPPING": { + "VIDEO": ["mp4", "webm"], + "AUDIO": ["mp3"], + "EXERCISE": ["perseus"], + "DOCUMENT": ["pdf", "epub", "bloompub", "bloomd"], + "HTML5": ["zip"], + "H5P": ["h5p"], + "ZIM": ["zim"] + } + } +} diff --git a/tests/test_kinds.py b/tests/test_kinds.py index 2ef3e27..3397814 100644 --- a/tests/test_kinds.py +++ b/tests/test_kinds.py @@ -1,18 +1,16 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -import json -import pkgutil - from le_utils.constants import content_kinds +from le_utils.constants import file_formats +from le_utils.constants import format_presets -def test_content_kind_extensions_are_synced(): - kindlookup = json.loads(pkgutil.get_data("le_utils", "resources/kindlookup.json").decode("utf-8")) - kinds_json = set(dict(kindlookup).keys()) - kinds_py = set(dict(content_kinds.choices).keys()) - assert kinds_json == kinds_py +def test_MAPPING_keys_are_file_format_ids(): + format_ids = {fmt.id for fmt in file_formats.FORMATLIST} + assert set(content_kinds.MAPPING) <= format_ids -def test_KINDLIST_exists(): - assert content_kinds.KINDLIST, "KINDLIST did not genereate properly" +def test_PRESETLIST_kinds_are_kind_ids(): + kind_ids = {kind.id for kind in content_kinds.KINDLIST} + assert {preset.kind for preset in format_presets.PRESETLIST if preset.kind} <= kind_ids