From dbcf95de0e245a403688a27ef591b499c7601d45 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Wed, 18 Nov 2026 18:22:12 +0600 Subject: [PATCH 1/2] lua: migrate first list batch onto shared cursor, IN and sort helpers --- .../lua/endpoints/getActorLikes.lua | 67 +++++--- .../lua/endpoints/getActorReposts.lua | 67 +++++--- .../lua/endpoints/getEntityFollow.lua | 24 ++- .../lua/endpoints/getEntityFollows.lua | 7 - happyview-modules/lua/endpoints/getLikes.lua | 67 +++++--- .../lua/endpoints/getReposts.lua | 67 +++++--- .../lua/endpoints/listActivities.lua | 93 ++++++----- .../lua/endpoints/listAttachments.lua | 102 +++++++----- .../lua/endpoints/listCollectionItems.lua | 21 ++- .../lua/endpoints/listCollections.lua | 151 ++++++++++-------- .../lua/endpoints/listEntityFollowers.lua | 42 +++-- .../lua/endpoints/listEntityFollowing.lua | 42 +++-- .../lua/endpoints/listEvaluations.lua | 102 +++++++----- .../lua/endpoints/listReceipts.lua | 115 +++++++------ .../lua/endpoints/searchActivities.lua | 93 ++++++----- .../lua/endpoints/searchBadgeDefinitions.lua | 12 +- .../lua/endpoints/searchCollections.lua | 151 ++++++++++-------- happyview-modules/lua/shared/activityList.lua | 46 +----- .../lua/shared/attachmentList.lua | 51 ++---- .../lua/shared/collectionItems.lua | 13 +- .../lua/shared/collectionList.lua | 75 ++------- happyview-modules/lua/shared/entityFollow.lua | 7 - .../lua/shared/entityFollowLookup.lua | 2 +- .../lua/shared/entityFollowPagination.lua | 24 +-- .../lua/shared/evaluationList.lua | 51 +----- happyview-modules/lua/shared/feedReaction.lua | 30 +--- .../lua/shared/fundingReceiptList.lua | 53 +----- .../lua/src/searchBadgeDefinitions.lua | 5 - .../modules/activity/manifest.json | 6 + .../modules/badge-queries/manifest.json | 1 + .../modules/collection/manifest.json | 11 ++ .../modules/context-attachments/manifest.json | 3 + .../modules/context-evaluation/manifest.json | 3 + .../modules/entity-follow/manifest.json | 5 + .../modules/feed-reactions/manifest.json | 8 + .../modules/funding/manifest.json | 4 + 36 files changed, 823 insertions(+), 798 deletions(-) diff --git a/happyview-modules/lua/endpoints/getActorLikes.lua b/happyview-modules/lua/endpoints/getActorLikes.lua index 11862f04..6973b374 100644 --- a/happyview-modules/lua/endpoints/getActorLikes.lua +++ b/happyview-modules/lua/endpoints/getActorLikes.lua @@ -124,6 +124,43 @@ local function hydrate_actor_views(actors, run_query) end end +-- Decodes a hex-encoded {v, d, t, u} list cursor. Rejects malformed tokens, other +-- sort directions and unknown keys; validate_value(value) -> boolean runs the +-- endpoint-specific check on t and u. Not for endpoints that skip the 8192-character +-- cap or accept extra keys. +local function decode_cursor(token, direction, validate_value) + if token == nil then return nil end + if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then + invalid("cursor is malformed") + end + local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) + local ok, value = pcall(json.decode, decoded) + if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction + or type(value.t) ~= "string" or type(value.u) ~= "string" then + invalid("cursor is malformed or belongs to another sortDirection") + end + for key in pairs(value) do + if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end + end + if not validate_value(value) then invalid("cursor is malformed") end + return value +end + +local CREATED_AT_SORT_PATTERN = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" + +-- Sort key for a happyview_records row: the record's own createdAt when it is a +-- valid timestamp, otherwise COALESCE(indexed_at, created_at). `alias` is the +-- table alias; nil selects unqualified columns. +local function created_at_sort(alias) + local prefix = alias and (alias .. ".") or "" + local record, indexed_at, created_at = prefix .. "record", prefix .. "indexed_at", prefix .. "created_at" + local created = record .. "::jsonb->>'createdAt'" + return "CASE WHEN jsonb_typeof(" .. record .. "::jsonb->'createdAt') = 'string' AND " .. created .. + " ~ '" .. CREATED_AT_SORT_PATTERN .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. + ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(" .. indexed_at .. + "::timestamptz, " .. created_at .. "::timestamptz) END" +end + local FEED_REACTIONS = { likes = { collection = "app.certified.feed.like", @@ -139,12 +176,6 @@ local FEED_REACTIONS = { }, } -local function feed_reaction_sort_key() - local created = "record::jsonb->>'createdAt'" - local zoned = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" - return "CASE WHEN jsonb_typeof(record::jsonb->'createdAt') = 'string' AND " .. created .. " ~ '" .. zoned .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(indexed_at::timestamptz, created_at::timestamptz) END" -end - local function feed_reaction_query(reaction, sql, values) if db.backend() ~= "postgres" then error(reaction.failure .. ": reaction queries require PostgreSQL", 0) end local ok, result = pcall(db.raw, sql, values) @@ -153,24 +184,10 @@ local function feed_reaction_query(reaction, sql, values) end local function feed_reaction_decode_cursor(token, direction, collection) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end - end - local valid, cursor_collection = valid_record_uri(value.u) - if not valid or cursor_collection ~= collection or not valid_datetime(value.t) then - invalid("cursor is malformed") - end - return value + return decode_cursor(token, direction, function(value) + local valid, cursor_collection = valid_record_uri(value.u) + return valid and cursor_collection == collection and valid_datetime(value.t) + end) end local function feed_reaction_query_page(reaction, mode, identity, limit, cursor, direction) @@ -196,7 +213,7 @@ local function feed_reaction_query_page(reaction, mode, identity, limit, cursor, local ordering = direction == "asc" and "ASC" or "DESC" local subject_uri = "record::jsonb->'subject'->>'uri'" -- Select the earliest representative before applying the caller's independent page direction. - local sql = "WITH ranked_reactions AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, " .. subject_uri .. " ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. feed_reaction_sort_key() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. "), " .. + local sql = "WITH ranked_reactions AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, " .. subject_uri .. " ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. created_at_sort() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. "), " .. "representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at FROM ranked_reactions WHERE relationship_rank = 1), " local page = "page AS (SELECT uri, did, cid, indexed_at, record, sort_at FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values .. ") " diff --git a/happyview-modules/lua/endpoints/getActorReposts.lua b/happyview-modules/lua/endpoints/getActorReposts.lua index 3c367fe7..84fb24ed 100644 --- a/happyview-modules/lua/endpoints/getActorReposts.lua +++ b/happyview-modules/lua/endpoints/getActorReposts.lua @@ -124,6 +124,43 @@ local function hydrate_actor_views(actors, run_query) end end +-- Decodes a hex-encoded {v, d, t, u} list cursor. Rejects malformed tokens, other +-- sort directions and unknown keys; validate_value(value) -> boolean runs the +-- endpoint-specific check on t and u. Not for endpoints that skip the 8192-character +-- cap or accept extra keys. +local function decode_cursor(token, direction, validate_value) + if token == nil then return nil end + if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then + invalid("cursor is malformed") + end + local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) + local ok, value = pcall(json.decode, decoded) + if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction + or type(value.t) ~= "string" or type(value.u) ~= "string" then + invalid("cursor is malformed or belongs to another sortDirection") + end + for key in pairs(value) do + if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end + end + if not validate_value(value) then invalid("cursor is malformed") end + return value +end + +local CREATED_AT_SORT_PATTERN = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" + +-- Sort key for a happyview_records row: the record's own createdAt when it is a +-- valid timestamp, otherwise COALESCE(indexed_at, created_at). `alias` is the +-- table alias; nil selects unqualified columns. +local function created_at_sort(alias) + local prefix = alias and (alias .. ".") or "" + local record, indexed_at, created_at = prefix .. "record", prefix .. "indexed_at", prefix .. "created_at" + local created = record .. "::jsonb->>'createdAt'" + return "CASE WHEN jsonb_typeof(" .. record .. "::jsonb->'createdAt') = 'string' AND " .. created .. + " ~ '" .. CREATED_AT_SORT_PATTERN .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. + ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(" .. indexed_at .. + "::timestamptz, " .. created_at .. "::timestamptz) END" +end + local FEED_REACTIONS = { likes = { collection = "app.certified.feed.like", @@ -139,12 +176,6 @@ local FEED_REACTIONS = { }, } -local function feed_reaction_sort_key() - local created = "record::jsonb->>'createdAt'" - local zoned = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" - return "CASE WHEN jsonb_typeof(record::jsonb->'createdAt') = 'string' AND " .. created .. " ~ '" .. zoned .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(indexed_at::timestamptz, created_at::timestamptz) END" -end - local function feed_reaction_query(reaction, sql, values) if db.backend() ~= "postgres" then error(reaction.failure .. ": reaction queries require PostgreSQL", 0) end local ok, result = pcall(db.raw, sql, values) @@ -153,24 +184,10 @@ local function feed_reaction_query(reaction, sql, values) end local function feed_reaction_decode_cursor(token, direction, collection) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end - end - local valid, cursor_collection = valid_record_uri(value.u) - if not valid or cursor_collection ~= collection or not valid_datetime(value.t) then - invalid("cursor is malformed") - end - return value + return decode_cursor(token, direction, function(value) + local valid, cursor_collection = valid_record_uri(value.u) + return valid and cursor_collection == collection and valid_datetime(value.t) + end) end local function feed_reaction_query_page(reaction, mode, identity, limit, cursor, direction) @@ -196,7 +213,7 @@ local function feed_reaction_query_page(reaction, mode, identity, limit, cursor, local ordering = direction == "asc" and "ASC" or "DESC" local subject_uri = "record::jsonb->'subject'->>'uri'" -- Select the earliest representative before applying the caller's independent page direction. - local sql = "WITH ranked_reactions AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, " .. subject_uri .. " ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. feed_reaction_sort_key() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. "), " .. + local sql = "WITH ranked_reactions AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, " .. subject_uri .. " ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. created_at_sort() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. "), " .. "representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at FROM ranked_reactions WHERE relationship_rank = 1), " local page = "page AS (SELECT uri, did, cid, indexed_at, record, sort_at FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values .. ") " diff --git a/happyview-modules/lua/endpoints/getEntityFollow.lua b/happyview-modules/lua/endpoints/getEntityFollow.lua index 5b98f157..5ba4bf8d 100644 --- a/happyview-modules/lua/endpoints/getEntityFollow.lua +++ b/happyview-modules/lua/endpoints/getEntityFollow.lua @@ -53,13 +53,6 @@ local function entity_follow_query(sql, values) return result end --- Keep malformed publisher timestamps out of casts; preserve the actor-follow fallback order. -local function entity_follow_sort_key() - local created = "record::jsonb->>'createdAt'" - local zoned = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" - return "CASE WHEN jsonb_typeof(record::jsonb->'createdAt') = 'string' AND " .. created .. " ~ '" .. zoned .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(indexed_at::timestamptz, created_at::timestamptz) END" -end - local function entity_follow_indexed_record_view(row) return { uri = row.uri, @@ -76,9 +69,24 @@ local function entity_follow_record_view(row) return view end +local CREATED_AT_SORT_PATTERN = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" + +-- Sort key for a happyview_records row: the record's own createdAt when it is a +-- valid timestamp, otherwise COALESCE(indexed_at, created_at). `alias` is the +-- table alias; nil selects unqualified columns. +local function created_at_sort(alias) + local prefix = alias and (alias .. ".") or "" + local record, indexed_at, created_at = prefix .. "record", prefix .. "indexed_at", prefix .. "created_at" + local created = record .. "::jsonb->>'createdAt'" + return "CASE WHEN jsonb_typeof(" .. record .. "::jsonb->'createdAt') = 'string' AND " .. created .. + " ~ '" .. CREATED_AT_SORT_PATTERN .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. + ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(" .. indexed_at .. + "::timestamptz, " .. created_at .. "::timestamptz) END" +end + local function entity_follow_lookup(actor, entity_uri) local rows = entity_follow_query( - "SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records CROSS JOIN LATERAL (SELECT " .. entity_follow_sort_key() .. " AS sort_at) sorted WHERE collection = $1 AND did = $2 AND record::jsonb->'subject'->>'uri' = $3 ORDER BY sorted.sort_at ASC, uri ASC LIMIT 1", + "SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records CROSS JOIN LATERAL (SELECT " .. created_at_sort() .. " AS sort_at) sorted WHERE collection = $1 AND did = $2 AND record::jsonb->'subject'->>'uri' = $3 ORDER BY sorted.sort_at ASC, uri ASC LIMIT 1", { ENTITY_FOLLOW, actor, entity_uri }) if #rows == 0 then return ENTITY_FOLLOW_NULL end return entity_follow_record_view(rows[1]) diff --git a/happyview-modules/lua/endpoints/getEntityFollows.lua b/happyview-modules/lua/endpoints/getEntityFollows.lua index 2356aa09..61cdc76d 100644 --- a/happyview-modules/lua/endpoints/getEntityFollows.lua +++ b/happyview-modules/lua/endpoints/getEntityFollows.lua @@ -53,13 +53,6 @@ local function entity_follow_query(sql, values) return result end --- Keep malformed publisher timestamps out of casts; preserve the actor-follow fallback order. -local function entity_follow_sort_key() - local created = "record::jsonb->>'createdAt'" - local zoned = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" - return "CASE WHEN jsonb_typeof(record::jsonb->'createdAt') = 'string' AND " .. created .. " ~ '" .. zoned .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(indexed_at::timestamptz, created_at::timestamptz) END" -end - local function entity_follow_indexed_record_view(row) return { uri = row.uri, diff --git a/happyview-modules/lua/endpoints/getLikes.lua b/happyview-modules/lua/endpoints/getLikes.lua index 3ac73cc7..81d7fa6f 100644 --- a/happyview-modules/lua/endpoints/getLikes.lua +++ b/happyview-modules/lua/endpoints/getLikes.lua @@ -124,6 +124,43 @@ local function hydrate_actor_views(actors, run_query) end end +-- Decodes a hex-encoded {v, d, t, u} list cursor. Rejects malformed tokens, other +-- sort directions and unknown keys; validate_value(value) -> boolean runs the +-- endpoint-specific check on t and u. Not for endpoints that skip the 8192-character +-- cap or accept extra keys. +local function decode_cursor(token, direction, validate_value) + if token == nil then return nil end + if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then + invalid("cursor is malformed") + end + local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) + local ok, value = pcall(json.decode, decoded) + if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction + or type(value.t) ~= "string" or type(value.u) ~= "string" then + invalid("cursor is malformed or belongs to another sortDirection") + end + for key in pairs(value) do + if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end + end + if not validate_value(value) then invalid("cursor is malformed") end + return value +end + +local CREATED_AT_SORT_PATTERN = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" + +-- Sort key for a happyview_records row: the record's own createdAt when it is a +-- valid timestamp, otherwise COALESCE(indexed_at, created_at). `alias` is the +-- table alias; nil selects unqualified columns. +local function created_at_sort(alias) + local prefix = alias and (alias .. ".") or "" + local record, indexed_at, created_at = prefix .. "record", prefix .. "indexed_at", prefix .. "created_at" + local created = record .. "::jsonb->>'createdAt'" + return "CASE WHEN jsonb_typeof(" .. record .. "::jsonb->'createdAt') = 'string' AND " .. created .. + " ~ '" .. CREATED_AT_SORT_PATTERN .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. + ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(" .. indexed_at .. + "::timestamptz, " .. created_at .. "::timestamptz) END" +end + local FEED_REACTIONS = { likes = { collection = "app.certified.feed.like", @@ -139,12 +176,6 @@ local FEED_REACTIONS = { }, } -local function feed_reaction_sort_key() - local created = "record::jsonb->>'createdAt'" - local zoned = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" - return "CASE WHEN jsonb_typeof(record::jsonb->'createdAt') = 'string' AND " .. created .. " ~ '" .. zoned .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(indexed_at::timestamptz, created_at::timestamptz) END" -end - local function feed_reaction_query(reaction, sql, values) if db.backend() ~= "postgres" then error(reaction.failure .. ": reaction queries require PostgreSQL", 0) end local ok, result = pcall(db.raw, sql, values) @@ -153,24 +184,10 @@ local function feed_reaction_query(reaction, sql, values) end local function feed_reaction_decode_cursor(token, direction, collection) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end - end - local valid, cursor_collection = valid_record_uri(value.u) - if not valid or cursor_collection ~= collection or not valid_datetime(value.t) then - invalid("cursor is malformed") - end - return value + return decode_cursor(token, direction, function(value) + local valid, cursor_collection = valid_record_uri(value.u) + return valid and cursor_collection == collection and valid_datetime(value.t) + end) end local function feed_reaction_query_page(reaction, mode, identity, limit, cursor, direction) @@ -196,7 +213,7 @@ local function feed_reaction_query_page(reaction, mode, identity, limit, cursor, local ordering = direction == "asc" and "ASC" or "DESC" local subject_uri = "record::jsonb->'subject'->>'uri'" -- Select the earliest representative before applying the caller's independent page direction. - local sql = "WITH ranked_reactions AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, " .. subject_uri .. " ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. feed_reaction_sort_key() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. "), " .. + local sql = "WITH ranked_reactions AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, " .. subject_uri .. " ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. created_at_sort() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. "), " .. "representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at FROM ranked_reactions WHERE relationship_rank = 1), " local page = "page AS (SELECT uri, did, cid, indexed_at, record, sort_at FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values .. ") " diff --git a/happyview-modules/lua/endpoints/getReposts.lua b/happyview-modules/lua/endpoints/getReposts.lua index 97880d21..98eed4e1 100644 --- a/happyview-modules/lua/endpoints/getReposts.lua +++ b/happyview-modules/lua/endpoints/getReposts.lua @@ -124,6 +124,43 @@ local function hydrate_actor_views(actors, run_query) end end +-- Decodes a hex-encoded {v, d, t, u} list cursor. Rejects malformed tokens, other +-- sort directions and unknown keys; validate_value(value) -> boolean runs the +-- endpoint-specific check on t and u. Not for endpoints that skip the 8192-character +-- cap or accept extra keys. +local function decode_cursor(token, direction, validate_value) + if token == nil then return nil end + if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then + invalid("cursor is malformed") + end + local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) + local ok, value = pcall(json.decode, decoded) + if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction + or type(value.t) ~= "string" or type(value.u) ~= "string" then + invalid("cursor is malformed or belongs to another sortDirection") + end + for key in pairs(value) do + if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end + end + if not validate_value(value) then invalid("cursor is malformed") end + return value +end + +local CREATED_AT_SORT_PATTERN = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" + +-- Sort key for a happyview_records row: the record's own createdAt when it is a +-- valid timestamp, otherwise COALESCE(indexed_at, created_at). `alias` is the +-- table alias; nil selects unqualified columns. +local function created_at_sort(alias) + local prefix = alias and (alias .. ".") or "" + local record, indexed_at, created_at = prefix .. "record", prefix .. "indexed_at", prefix .. "created_at" + local created = record .. "::jsonb->>'createdAt'" + return "CASE WHEN jsonb_typeof(" .. record .. "::jsonb->'createdAt') = 'string' AND " .. created .. + " ~ '" .. CREATED_AT_SORT_PATTERN .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. + ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(" .. indexed_at .. + "::timestamptz, " .. created_at .. "::timestamptz) END" +end + local FEED_REACTIONS = { likes = { collection = "app.certified.feed.like", @@ -139,12 +176,6 @@ local FEED_REACTIONS = { }, } -local function feed_reaction_sort_key() - local created = "record::jsonb->>'createdAt'" - local zoned = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" - return "CASE WHEN jsonb_typeof(record::jsonb->'createdAt') = 'string' AND " .. created .. " ~ '" .. zoned .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(indexed_at::timestamptz, created_at::timestamptz) END" -end - local function feed_reaction_query(reaction, sql, values) if db.backend() ~= "postgres" then error(reaction.failure .. ": reaction queries require PostgreSQL", 0) end local ok, result = pcall(db.raw, sql, values) @@ -153,24 +184,10 @@ local function feed_reaction_query(reaction, sql, values) end local function feed_reaction_decode_cursor(token, direction, collection) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end - end - local valid, cursor_collection = valid_record_uri(value.u) - if not valid or cursor_collection ~= collection or not valid_datetime(value.t) then - invalid("cursor is malformed") - end - return value + return decode_cursor(token, direction, function(value) + local valid, cursor_collection = valid_record_uri(value.u) + return valid and cursor_collection == collection and valid_datetime(value.t) + end) end local function feed_reaction_query_page(reaction, mode, identity, limit, cursor, direction) @@ -196,7 +213,7 @@ local function feed_reaction_query_page(reaction, mode, identity, limit, cursor, local ordering = direction == "asc" and "ASC" or "DESC" local subject_uri = "record::jsonb->'subject'->>'uri'" -- Select the earliest representative before applying the caller's independent page direction. - local sql = "WITH ranked_reactions AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, " .. subject_uri .. " ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. feed_reaction_sort_key() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. "), " .. + local sql = "WITH ranked_reactions AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, " .. subject_uri .. " ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. created_at_sort() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. "), " .. "representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at FROM ranked_reactions WHERE relationship_rank = 1), " local page = "page AS (SELECT uri, did, cid, indexed_at, record, sort_at FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values .. ") " diff --git a/happyview-modules/lua/endpoints/listActivities.lua b/happyview-modules/lua/endpoints/listActivities.lua index 77d6a7a0..5a8060ec 100644 --- a/happyview-modules/lua/endpoints/listActivities.lua +++ b/happyview-modules/lua/endpoints/listActivities.lua @@ -283,6 +283,53 @@ local function created_at_sort(alias) "::timestamptz, " .. created_at .. "::timestamptz) END" end +local function parse_sort_direction(params) + local direction = scalar(params, "sortDirection") or "desc" + if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + return direction +end + +local function cursor_encode(value) + local encoded = json.encode(value) + return (encoded:gsub(".", function(char) return string.format("%02x", string.byte(char)) end)) +end + +-- Decodes a hex-encoded {v, d, t, u} list cursor. Rejects malformed tokens, other +-- sort directions and unknown keys; validate_value(value) -> boolean runs the +-- endpoint-specific check on t and u. Not for endpoints that skip the 8192-character +-- cap or accept extra keys. +local function decode_cursor(token, direction, validate_value) + if token == nil then return nil end + if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then + invalid("cursor is malformed") + end + local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) + local ok, value = pcall(json.decode, decoded) + if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction + or type(value.t) ~= "string" or type(value.u) ~= "string" then + invalid("cursor is malformed or belongs to another sortDirection") + end + for key in pairs(value) do + if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end + end + if not validate_value(value) then invalid("cursor is malformed") end + return value +end + +-- Adds `column IN ($n, ...)` to where, binding items; an empty list matches nothing. +local function add_in(where, binds, column, items) + if #items == 0 then + where[#where + 1] = "FALSE" + return + end + local marks = {} + for _, item in ipairs(items) do + binds[#binds + 1] = item + marks[#marks + 1] = "$" .. #binds + end + where[#where + 1] = column .. " IN (" .. table.concat(marks, ", ") .. ")" +end + local function activity_scalar(values, key) if key == "hasOrganizationRecord" and type(values[key]) == "boolean" then return tostring(values[key]) @@ -333,30 +380,10 @@ local function activity_array(key, format) end local function decode_activity_cursor(token, direction) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end - end - local valid, collection = valid_record_uri(value.u) - if not valid_datetime(value.t) or not valid or collection ~= ACTIVITY then - invalid("cursor is malformed") - end - return value -end - -local function encode_activity_cursor(value) - return (json.encode(value):gsub(".", function(char) - return string.format("%02x", string.byte(char)) - end)) + return decode_cursor(token, direction, function(value) + local valid, collection = valid_record_uri(value.u) + return valid_datetime(value.t) and valid and collection == ACTIVITY + end) end local function bind_values(values, items) @@ -387,13 +414,7 @@ end local function activity_query(authors, has_organization_record, contributors, involved_actors, uris, search, limit, cursor, direction) local where, values = { "activity.collection = $1" }, { ACTIVITY } - if authors then - if #authors == 0 then - where[#where + 1] = "FALSE" - else - where[#where + 1] = "activity.did IN (" .. table.concat(bind_values(values, authors), ", ") .. ")" - end - end + if authors then add_in(where, values, "activity.did", authors) end if has_organization_record ~= nil then local predicate = has_organization_record and "EXISTS" or "NOT EXISTS" where[#where + 1] = predicate .. " (SELECT 1 FROM happyview_records AS organization " .. @@ -413,10 +434,7 @@ local function activity_query(authors, has_organization_record, contributors, in activity_contributor_match(values, involved_actors) .. ")" end end - if uris then - if #uris == 0 then where[#where + 1] = "FALSE" - else where[#where + 1] = "activity.uri IN (" .. table.concat(bind_values(values, uris), ", ") .. ")" end - end + if uris then add_in(where, values, "activity.uri", uris) end if search ~= nil then values[#values + 1] = search local pattern = "$" .. #values @@ -452,7 +470,7 @@ local function activity_query(authors, has_organization_record, contributors, in local next_cursor if more then local last = rows[#rows] - next_cursor = encode_activity_cursor({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) + next_cursor = cursor_encode({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) end return views, next_cursor end @@ -492,8 +510,7 @@ local function activity_list_response(search_enabled) search = search:match("^%s*(.-)%s*$") if search == "" then search = nil end end - local direction = scalar(params, "sortDirection") or "desc" - if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + local direction = parse_sort_direction(params) local limit = parse_list_limit(params) local cursor = decode_activity_cursor(scalar(params, "cursor"), direction) local activities, next_cursor = activity_query( diff --git a/happyview-modules/lua/endpoints/listAttachments.lua b/happyview-modules/lua/endpoints/listAttachments.lua index e5c93c7f..e4349b60 100644 --- a/happyview-modules/lua/endpoints/listAttachments.lua +++ b/happyview-modules/lua/endpoints/listAttachments.lua @@ -158,6 +158,57 @@ local function hydrate_attachment_views(views) hydrate_actor_views(authors, attachment_query) end +-- Decodes a hex-encoded {v, d, t, u} list cursor. Rejects malformed tokens, other +-- sort directions and unknown keys; validate_value(value) -> boolean runs the +-- endpoint-specific check on t and u. Not for endpoints that skip the 8192-character +-- cap or accept extra keys. +local function decode_cursor(token, direction, validate_value) + if token == nil then return nil end + if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then + invalid("cursor is malformed") + end + local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) + local ok, value = pcall(json.decode, decoded) + if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction + or type(value.t) ~= "string" or type(value.u) ~= "string" then + invalid("cursor is malformed or belongs to another sortDirection") + end + for key in pairs(value) do + if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end + end + if not validate_value(value) then invalid("cursor is malformed") end + return value +end + +-- Adds `column IN ($n, ...)` to where, binding items; an empty list matches nothing. +local function add_in(where, binds, column, items) + if #items == 0 then + where[#where + 1] = "FALSE" + return + end + local marks = {} + for _, item in ipairs(items) do + binds[#binds + 1] = item + marks[#marks + 1] = "$" .. #binds + end + where[#where + 1] = column .. " IN (" .. table.concat(marks, ", ") .. ")" +end + +local CREATED_AT_SORT_PATTERN = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" + +-- Sort key for a happyview_records row: the record's own createdAt when it is a +-- valid timestamp, otherwise COALESCE(indexed_at, created_at). `alias` is the +-- table alias; nil selects unqualified columns. +local function created_at_sort(alias) + local prefix = alias and (alias .. ".") or "" + local record, indexed_at, created_at = prefix .. "record", prefix .. "indexed_at", prefix .. "created_at" + local created = record .. "::jsonb->>'createdAt'" + return "CASE WHEN jsonb_typeof(" .. record .. "::jsonb->'createdAt') = 'string' AND " .. created .. + " ~ '" .. CREATED_AT_SORT_PATTERN .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. + ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(" .. indexed_at .. + "::timestamptz, " .. created_at .. "::timestamptz) END" +end + local function attachment_array(params, key, validate, description) local value = params[key] if value == nil then return nil end @@ -193,20 +244,6 @@ local function attachment_array(params, key, validate, description) return unique end -local function attachment_bind_in(where, values, expression, items) - if not items then return end - if #items == 0 then - where[#where + 1] = "FALSE" - return - end - local placeholders = {} - for _, item in ipairs(items) do - values[#values + 1] = item - placeholders[#placeholders + 1] = "$" .. #values - end - where[#where + 1] = expression .. " IN (" .. table.concat(placeholders, ", ") .. ")" -end - local function attachment_subject_filter(where, values, subjects) if not subjects then return end if #subjects == 0 then @@ -224,39 +261,20 @@ local function attachment_subject_filter(where, values, subjects) "WHERE subject.value->>'uri' IN (" .. table.concat(placeholders, ", ") .. "))" end -local function attachment_sort_expression() - local created = "attachment.record::jsonb->>'createdAt'" - local zoned = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" - return "CASE WHEN jsonb_typeof(attachment.record::jsonb->'createdAt') = 'string' AND " .. created .. - " ~ '" .. zoned .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. - ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE " .. - "COALESCE(attachment.indexed_at::timestamptz, attachment.created_at::timestamptz) END" -end - local function attachment_cursor_decode(token, direction) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end - end - if not valid_datetime(value.t) or not valid_attachment_uri(value.u) then invalid("cursor is malformed") end - return value + return decode_cursor(token, direction, function(value) + return valid_datetime(value.t) and valid_attachment_uri(value.u) + end) end local function query_attachments(filters, limit, cursor, direction) local where, values = { "attachment.collection = $1" }, { ATTACHMENT_COLLECTION } - attachment_bind_in(where, values, "attachment.did", filters.authors) - attachment_bind_in(where, values, "attachment.uri", filters.uris) + if filters.authors then add_in(where, values, "attachment.did", filters.authors) end + if filters.uris then add_in(where, values, "attachment.uri", filters.uris) end attachment_subject_filter(where, values, filters.subjects) - attachment_bind_in(where, values, "attachment.record::jsonb->>'contentType'", filters.content_types) + if filters.content_types then + add_in(where, values, "attachment.record::jsonb->>'contentType'", filters.content_types) + end if cursor then values[#values + 1] = cursor.t @@ -270,7 +288,7 @@ local function query_attachments(filters, limit, cursor, direction) values[#values + 1] = limit + 1 local ordering = direction == "asc" and "ASC" or "DESC" - local sort_key = attachment_sort_expression() + local sort_key = created_at_sort("attachment") local sql = "SELECT attachment.uri, attachment.did, attachment.cid, attachment.indexed_at::text AS indexed_at, " .. "attachment.record::text AS record, to_char(sorted.sort_at AT TIME ZONE 'UTC', " .. "'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp " .. diff --git a/happyview-modules/lua/endpoints/listCollectionItems.lua b/happyview-modules/lua/endpoints/listCollectionItems.lua index 68d559f0..06d1f5b9 100644 --- a/happyview-modules/lua/endpoints/listCollectionItems.lua +++ b/happyview-modules/lua/endpoints/listCollectionItems.lua @@ -287,6 +287,14 @@ local function feature_projection_hydrate(views) end end +local function parse_list_limit(params) + local limit_value = scalar(params, "limit") + if limit_value and not limit_value:match("^%d+$") then invalid("limit must be an integer from 1 through 100") end + local limit = limit_value and tonumber(limit_value) or 25 + if not limit or limit % 1 ~= 0 or limit < 1 or limit > 100 then invalid("limit must be an integer from 1 through 100") end + return limit +end + local ITEM_COLLECTION = "org.hypercerts.collection" local ITEM_ACTIVITY = "org.hypercerts.claim.activity" local ITEM_FEATURE = "org.hypercerts.entity.feature" @@ -300,17 +308,6 @@ local function collection_items_keys_only(values, allowed) keys_only(values, allowed, "unknown query parameter: ") end -local function collection_items_limit() - local value = scalar(params, "limit") - if value == nil then return 25 end - if not value:match("^%d+$") then collection_items_invalid("limit must be an integer from 1 through 100") end - local limit = tonumber(value) - if not limit or limit < 1 or limit > 100 then - collection_items_invalid("limit must be an integer from 1 through 100") - end - return limit -end - local function collection_items_cursor_decode(token, collection_uri) if token == nil then return nil end if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then @@ -454,7 +451,7 @@ local function collection_items_response() if not collection_uri or not valid or collection ~= ITEM_COLLECTION then collection_items_invalid("collection must be a full org.hypercerts.collection AT-URI with a DID authority") end - local limit = collection_items_limit() + local limit = parse_list_limit(params) local cursor = collection_items_cursor_decode(scalar(params, "cursor"), collection_uri) local collection_rows = collection_items_query( diff --git a/happyview-modules/lua/endpoints/listCollections.lua b/happyview-modules/lua/endpoints/listCollections.lua index a23780bb..a0b04803 100644 --- a/happyview-modules/lua/endpoints/listCollections.lua +++ b/happyview-modules/lua/endpoints/listCollections.lua @@ -291,6 +291,82 @@ local function created_at_sort(alias) "::timestamptz, " .. created_at .. "::timestamptz) END" end +local function invalid(message) + error("InvalidRequest: " .. message, 0) +end + +local function keys_only(values, allowed, unknown_message_prefix) + for key in pairs(values) do + if not allowed[key] then + invalid(unknown_message_prefix and (unknown_message_prefix .. key) or "unknown query parameter") + end + end +end + +local function scalar(params, key) + local value = params[key] + if value == nil then return nil end + if type(value) ~= "string" and type(value) ~= "number" then + invalid(key .. " must occur once") + end + return tostring(value) +end + +local function parse_list_limit(params) + local limit_value = scalar(params, "limit") + if limit_value and not limit_value:match("^%d+$") then invalid("limit must be an integer from 1 through 100") end + local limit = limit_value and tonumber(limit_value) or 25 + if not limit or limit % 1 ~= 0 or limit < 1 or limit > 100 then invalid("limit must be an integer from 1 through 100") end + return limit +end + +local function parse_sort_direction(params) + local direction = scalar(params, "sortDirection") or "desc" + if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + return direction +end + +local function cursor_encode(value) + local encoded = json.encode(value) + return (encoded:gsub(".", function(char) return string.format("%02x", string.byte(char)) end)) +end + +-- Decodes a hex-encoded {v, d, t, u} list cursor. Rejects malformed tokens, other +-- sort directions and unknown keys; validate_value(value) -> boolean runs the +-- endpoint-specific check on t and u. Not for endpoints that skip the 8192-character +-- cap or accept extra keys. +local function decode_cursor(token, direction, validate_value) + if token == nil then return nil end + if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then + invalid("cursor is malformed") + end + local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) + local ok, value = pcall(json.decode, decoded) + if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction + or type(value.t) ~= "string" or type(value.u) ~= "string" then + invalid("cursor is malformed or belongs to another sortDirection") + end + for key in pairs(value) do + if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end + end + if not validate_value(value) then invalid("cursor is malformed") end + return value +end + +-- Adds `column IN ($n, ...)` to where, binding items; an empty list matches nothing. +local function add_in(where, binds, column, items) + if #items == 0 then + where[#where + 1] = "FALSE" + return + end + local marks = {} + for _, item in ipairs(items) do + binds[#binds + 1] = item + marks[#marks + 1] = "$" .. #binds + end + where[#where + 1] = column .. " IN (" .. table.concat(marks, ", ") .. ")" +end + local function collection_array(key, kind) local value = params[key] if value == nil then return nil end @@ -345,44 +421,11 @@ local function collection_array(key, kind) return unique end -local function collection_list_limit() - local value = collection_scalar(params, "limit") - if value == nil then return 25 end - if not value:match("^%d+$") then collection_invalid("limit must be an integer from 1 through 100") end - local limit = tonumber(value) - if not limit or limit < 1 or limit > 100 then - collection_invalid("limit must be an integer from 1 through 100") - end - return limit -end - local function collection_list_cursor_decode(token, direction) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - collection_invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - collection_invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then - collection_invalid("cursor is malformed") - end - end - local valid, collection = collection_valid_record_uri(value.u) - if not valid_datetime(value.t) or not valid or collection ~= COLLECTION then - collection_invalid("cursor is malformed") - end - return value -end - -local function collection_cursor_encode(value) - return (json.encode(value):gsub(".", function(char) - return string.format("%02x", string.byte(char)) - end)) + return decode_cursor(token, direction, function(value) + local valid, collection = collection_valid_record_uri(value.u) + return valid_datetime(value.t) and valid and collection == COLLECTION + end) end local function collection_bind_values(values, items) @@ -397,34 +440,15 @@ end local function collection_list_filters(filters, search) local where, values = { "collection.collection = $1" }, { COLLECTION } - if filters.authors then - if #filters.authors == 0 then - where[#where + 1] = "FALSE" - else - where[#where + 1] = "collection.did IN (" .. table.concat(collection_bind_values(values, filters.authors), ", ") .. ")" - end - end + if filters.authors then add_in(where, values, "collection.did", filters.authors) end if filters.hasOrganizationRecord ~= nil then local predicate = filters.hasOrganizationRecord and "EXISTS" or "NOT EXISTS" where[#where + 1] = predicate .. " (SELECT 1 FROM happyview_records AS organization " .. "WHERE organization.collection = 'app.certified.actor.organization' AND organization.rkey = 'self' " .. "AND organization.did = collection.did)" end - if filters.types then - if #filters.types == 0 then - where[#where + 1] = "FALSE" - else - where[#where + 1] = "collection.record::jsonb->>'type' IN (" .. - table.concat(collection_bind_values(values, filters.types), ", ") .. ")" - end - end - if filters.uris then - if #filters.uris == 0 then - where[#where + 1] = "FALSE" - else - where[#where + 1] = "collection.uri IN (" .. table.concat(collection_bind_values(values, filters.uris), ", ") .. ")" - end - end + if filters.types then add_in(where, values, "collection.record::jsonb->>'type'", filters.types) end + if filters.uris then add_in(where, values, "collection.uri", filters.uris) end if filters.itemUris then if #filters.itemUris == 0 then where[#where + 1] = "FALSE" @@ -491,7 +515,7 @@ local function collection_list_page(rows, limit, direction) local next_cursor if more then local last = rows[#rows] - next_cursor = collection_cursor_encode({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) + next_cursor = cursor_encode({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) end return views, next_cursor end @@ -540,11 +564,8 @@ local function collection_list_response(search_enabled) search = search:match("^%s*(.-)%s*$") if search == "" then search = nil end end - local direction = collection_scalar(params, "sortDirection") or "desc" - if direction ~= "asc" and direction ~= "desc" then - collection_invalid("sortDirection must be 'asc' or 'desc'") - end - local limit = collection_list_limit() + local direction = parse_sort_direction(params) + local limit = parse_list_limit(params) local cursor = collection_list_cursor_decode(collection_scalar(params, "cursor"), direction) local collections, next_cursor = collection_list_query({ authors = authors, diff --git a/happyview-modules/lua/endpoints/listEntityFollowers.lua b/happyview-modules/lua/endpoints/listEntityFollowers.lua index 98385511..1a7e3620 100644 --- a/happyview-modules/lua/endpoints/listEntityFollowers.lua +++ b/happyview-modules/lua/endpoints/listEntityFollowers.lua @@ -96,13 +96,6 @@ local function entity_follow_query(sql, values) return result end --- Keep malformed publisher timestamps out of casts; preserve the actor-follow fallback order. -local function entity_follow_sort_key() - local created = "record::jsonb->>'createdAt'" - local zoned = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" - return "CASE WHEN jsonb_typeof(record::jsonb->'createdAt') = 'string' AND " .. created .. " ~ '" .. zoned .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(indexed_at::timestamptz, created_at::timestamptz) END" -end - local function entity_follow_indexed_record_view(row) return { uri = row.uri, @@ -119,7 +112,11 @@ local function entity_follow_record_view(row) return view end -local function entity_follow_decode_cursor(token, direction) +-- Decodes a hex-encoded {v, d, t, u} list cursor. Rejects malformed tokens, other +-- sort directions and unknown keys; validate_value(value) -> boolean runs the +-- endpoint-specific check on t and u. Not for endpoints that skip the 8192-character +-- cap or accept extra keys. +local function decode_cursor(token, direction, validate_value) if token == nil then return nil end if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then invalid("cursor is malformed") @@ -133,13 +130,32 @@ local function entity_follow_decode_cursor(token, direction) for key in pairs(value) do if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end end - local valid, collection = valid_record_uri(value.u) - if not valid or collection ~= ENTITY_FOLLOW or not valid_datetime(value.t) then - invalid("cursor is malformed") - end + if not validate_value(value) then invalid("cursor is malformed") end return value end +local CREATED_AT_SORT_PATTERN = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" + +-- Sort key for a happyview_records row: the record's own createdAt when it is a +-- valid timestamp, otherwise COALESCE(indexed_at, created_at). `alias` is the +-- table alias; nil selects unqualified columns. +local function created_at_sort(alias) + local prefix = alias and (alias .. ".") or "" + local record, indexed_at, created_at = prefix .. "record", prefix .. "indexed_at", prefix .. "created_at" + local created = record .. "::jsonb->>'createdAt'" + return "CASE WHEN jsonb_typeof(" .. record .. "::jsonb->'createdAt') = 'string' AND " .. created .. + " ~ '" .. CREATED_AT_SORT_PATTERN .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. + ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(" .. indexed_at .. + "::timestamptz, " .. created_at .. "::timestamptz) END" +end + +local function entity_follow_decode_cursor(token, direction) + return decode_cursor(token, direction, function(value) + local valid, collection = valid_record_uri(value.u) + return valid and collection == ENTITY_FOLLOW and valid_datetime(value.t) + end) +end + local function entity_follow_query_page(mode, identity, limit, cursor, direction) local filters, values = { "collection = $1" }, { ENTITY_FOLLOW, identity } if mode == "followers" then @@ -161,7 +177,7 @@ local function entity_follow_query_page(mode, identity, limit, cursor, direction values[#values + 1] = limit + 1 local ordering = direction == "asc" and "ASC" or "DESC" local subject_uri = "record::jsonb->'subject'->>'uri'" - local ranked_follows = "ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, " .. subject_uri .. " ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. entity_follow_sort_key() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. ")" + local ranked_follows = "ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, " .. subject_uri .. " ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. created_at_sort() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. ")" local representatives = "representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at FROM ranked_follows WHERE relationship_rank = 1)" local page = "page AS (SELECT uri, did, cid, indexed_at, record, sort_at FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values .. ")" local sql diff --git a/happyview-modules/lua/endpoints/listEntityFollowing.lua b/happyview-modules/lua/endpoints/listEntityFollowing.lua index e2c44d89..8e91ea3a 100644 --- a/happyview-modules/lua/endpoints/listEntityFollowing.lua +++ b/happyview-modules/lua/endpoints/listEntityFollowing.lua @@ -532,13 +532,6 @@ local function entity_follow_query(sql, values) return result end --- Keep malformed publisher timestamps out of casts; preserve the actor-follow fallback order. -local function entity_follow_sort_key() - local created = "record::jsonb->>'createdAt'" - local zoned = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" - return "CASE WHEN jsonb_typeof(record::jsonb->'createdAt') = 'string' AND " .. created .. " ~ '" .. zoned .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(indexed_at::timestamptz, created_at::timestamptz) END" -end - local function entity_follow_indexed_record_view(row) return { uri = row.uri, @@ -555,7 +548,11 @@ local function entity_follow_record_view(row) return view end -local function entity_follow_decode_cursor(token, direction) +-- Decodes a hex-encoded {v, d, t, u} list cursor. Rejects malformed tokens, other +-- sort directions and unknown keys; validate_value(value) -> boolean runs the +-- endpoint-specific check on t and u. Not for endpoints that skip the 8192-character +-- cap or accept extra keys. +local function decode_cursor(token, direction, validate_value) if token == nil then return nil end if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then invalid("cursor is malformed") @@ -569,13 +566,32 @@ local function entity_follow_decode_cursor(token, direction) for key in pairs(value) do if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end end - local valid, collection = valid_record_uri(value.u) - if not valid or collection ~= ENTITY_FOLLOW or not valid_datetime(value.t) then - invalid("cursor is malformed") - end + if not validate_value(value) then invalid("cursor is malformed") end return value end +local CREATED_AT_SORT_PATTERN = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" + +-- Sort key for a happyview_records row: the record's own createdAt when it is a +-- valid timestamp, otherwise COALESCE(indexed_at, created_at). `alias` is the +-- table alias; nil selects unqualified columns. +local function created_at_sort(alias) + local prefix = alias and (alias .. ".") or "" + local record, indexed_at, created_at = prefix .. "record", prefix .. "indexed_at", prefix .. "created_at" + local created = record .. "::jsonb->>'createdAt'" + return "CASE WHEN jsonb_typeof(" .. record .. "::jsonb->'createdAt') = 'string' AND " .. created .. + " ~ '" .. CREATED_AT_SORT_PATTERN .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. + ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(" .. indexed_at .. + "::timestamptz, " .. created_at .. "::timestamptz) END" +end + +local function entity_follow_decode_cursor(token, direction) + return decode_cursor(token, direction, function(value) + local valid, collection = valid_record_uri(value.u) + return valid and collection == ENTITY_FOLLOW and valid_datetime(value.t) + end) +end + local function entity_follow_query_page(mode, identity, limit, cursor, direction) local filters, values = { "collection = $1" }, { ENTITY_FOLLOW, identity } if mode == "followers" then @@ -597,7 +613,7 @@ local function entity_follow_query_page(mode, identity, limit, cursor, direction values[#values + 1] = limit + 1 local ordering = direction == "asc" and "ASC" or "DESC" local subject_uri = "record::jsonb->'subject'->>'uri'" - local ranked_follows = "ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, " .. subject_uri .. " ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. entity_follow_sort_key() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. ")" + local ranked_follows = "ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, " .. subject_uri .. " ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. created_at_sort() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. ")" local representatives = "representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at FROM ranked_follows WHERE relationship_rank = 1)" local page = "page AS (SELECT uri, did, cid, indexed_at, record, sort_at FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values .. ")" local sql diff --git a/happyview-modules/lua/endpoints/listEvaluations.lua b/happyview-modules/lua/endpoints/listEvaluations.lua index 3925cda6..eac07546 100644 --- a/happyview-modules/lua/endpoints/listEvaluations.lua +++ b/happyview-modules/lua/endpoints/listEvaluations.lua @@ -220,6 +220,57 @@ local function valid_nsid(value) return #name <= 63 and name:match("^[A-Za-z][A-Za-z0-9]*$") ~= nil end +-- Decodes a hex-encoded {v, d, t, u} list cursor. Rejects malformed tokens, other +-- sort directions and unknown keys; validate_value(value) -> boolean runs the +-- endpoint-specific check on t and u. Not for endpoints that skip the 8192-character +-- cap or accept extra keys. +local function decode_cursor(token, direction, validate_value) + if token == nil then return nil end + if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then + invalid("cursor is malformed") + end + local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) + local ok, value = pcall(json.decode, decoded) + if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction + or type(value.t) ~= "string" or type(value.u) ~= "string" then + invalid("cursor is malformed or belongs to another sortDirection") + end + for key in pairs(value) do + if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end + end + if not validate_value(value) then invalid("cursor is malformed") end + return value +end + +-- Adds `column IN ($n, ...)` to where, binding items; an empty list matches nothing. +local function add_in(where, binds, column, items) + if #items == 0 then + where[#where + 1] = "FALSE" + return + end + local marks = {} + for _, item in ipairs(items) do + binds[#binds + 1] = item + marks[#marks + 1] = "$" .. #binds + end + where[#where + 1] = column .. " IN (" .. table.concat(marks, ", ") .. ")" +end + +local CREATED_AT_SORT_PATTERN = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" + +-- Sort key for a happyview_records row: the record's own createdAt when it is a +-- valid timestamp, otherwise COALESCE(indexed_at, created_at). `alias` is the +-- table alias; nil selects unqualified columns. +local function created_at_sort(alias) + local prefix = alias and (alias .. ".") or "" + local record, indexed_at, created_at = prefix .. "record", prefix .. "indexed_at", prefix .. "created_at" + local created = record .. "::jsonb->>'createdAt'" + return "CASE WHEN jsonb_typeof(" .. record .. "::jsonb->'createdAt') = 'string' AND " .. created .. + " ~ '" .. CREATED_AT_SORT_PATTERN .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. + ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(" .. indexed_at .. + "::timestamptz, " .. created_at .. "::timestamptz) END" +end + local function evaluation_array(key, format) local value = params[key] if value == nil then return nil end @@ -280,46 +331,16 @@ local function evaluation_evaluator_match(values, dids) "WHERE evaluator.value->>'did' IN (" .. table.concat(marks, ", ") .. "))" end -local function evaluation_sort_expression() - local created = "evaluation.record::jsonb->>'createdAt'" - local zoned = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" - return "CASE WHEN jsonb_typeof(evaluation.record::jsonb->'createdAt') = 'string' AND " .. created .. - " ~ '" .. zoned .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. - ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE " .. - "COALESCE(evaluation.indexed_at::timestamptz, evaluation.created_at::timestamptz) END" -end - local function decode_evaluation_cursor(token, direction) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end - end - local valid, collection = valid_record_uri(value.u) - if not valid_datetime(value.t) or not valid or collection ~= EVALUATION then - invalid("cursor is malformed") - end - return value + return decode_cursor(token, direction, function(value) + local valid, collection = valid_record_uri(value.u) + return valid_datetime(value.t) and valid and collection == EVALUATION + end) end local function evaluation_list_filters(authors, evaluators, subjects) local where, values = { "evaluation.collection = $1" }, { EVALUATION } - if authors then - if #authors == 0 then - where[#where + 1] = "FALSE" - else - where[#where + 1] = "evaluation.did IN (" .. - table.concat(bind_evaluation_values(values, authors), ", ") .. ")" - end - end + if authors then add_in(where, values, "evaluation.did", authors) end if evaluators then if #evaluators == 0 then where[#where + 1] = "FALSE" @@ -327,14 +348,7 @@ local function evaluation_list_filters(authors, evaluators, subjects) where[#where + 1] = evaluation_evaluator_match(values, evaluators) end end - if subjects then - if #subjects == 0 then - where[#where + 1] = "FALSE" - else - where[#where + 1] = "evaluation.record::jsonb->'subject'->>'uri' IN (" .. - table.concat(bind_evaluation_values(values, subjects), ", ") .. ")" - end - end + if subjects then add_in(where, values, "evaluation.record::jsonb->'subject'->>'uri'", subjects) end return where, values end @@ -359,7 +373,7 @@ local function evaluation_list_page_query(values, where, cursor, limit, directio "to_char(sorted.sort_at AT TIME ZONE 'UTC', " .. "'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp " .. "FROM happyview_records AS evaluation CROSS JOIN LATERAL (SELECT " .. - evaluation_sort_expression() .. " AS sort_at) AS sorted WHERE " .. table.concat(page_where, " AND ") .. + created_at_sort("evaluation") .. " AS sort_at) AS sorted WHERE " .. table.concat(page_where, " AND ") .. " ORDER BY sorted.sort_at " .. ordering .. ", evaluation.uri " .. ordering .. " LIMIT $" .. #page_values return sql, page_values end diff --git a/happyview-modules/lua/endpoints/listReceipts.lua b/happyview-modules/lua/endpoints/listReceipts.lua index 52315b75..e798215b 100644 --- a/happyview-modules/lua/endpoints/listReceipts.lua +++ b/happyview-modules/lua/endpoints/listReceipts.lua @@ -165,6 +165,68 @@ local function valid_nsid(value) return #name <= 63 and name:match("^[A-Za-z][A-Za-z0-9]*$") ~= nil end +local function parse_sort_direction(params) + local direction = scalar(params, "sortDirection") or "desc" + if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + return direction +end + +local function cursor_encode(value) + local encoded = json.encode(value) + return (encoded:gsub(".", function(char) return string.format("%02x", string.byte(char)) end)) +end + +-- Decodes a hex-encoded {v, d, t, u} list cursor. Rejects malformed tokens, other +-- sort directions and unknown keys; validate_value(value) -> boolean runs the +-- endpoint-specific check on t and u. Not for endpoints that skip the 8192-character +-- cap or accept extra keys. +local function decode_cursor(token, direction, validate_value) + if token == nil then return nil end + if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then + invalid("cursor is malformed") + end + local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) + local ok, value = pcall(json.decode, decoded) + if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction + or type(value.t) ~= "string" or type(value.u) ~= "string" then + invalid("cursor is malformed or belongs to another sortDirection") + end + for key in pairs(value) do + if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end + end + if not validate_value(value) then invalid("cursor is malformed") end + return value +end + +-- Adds `column IN ($n, ...)` to where, binding items; an empty list matches nothing. +local function add_in(where, binds, column, items) + if #items == 0 then + where[#where + 1] = "FALSE" + return + end + local marks = {} + for _, item in ipairs(items) do + binds[#binds + 1] = item + marks[#marks + 1] = "$" .. #binds + end + where[#where + 1] = column .. " IN (" .. table.concat(marks, ", ") .. ")" +end + +local CREATED_AT_SORT_PATTERN = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" + +-- Sort key for a happyview_records row: the record's own createdAt when it is a +-- valid timestamp, otherwise COALESCE(indexed_at, created_at). `alias` is the +-- table alias; nil selects unqualified columns. +local function created_at_sort(alias) + local prefix = alias and (alias .. ".") or "" + local record, indexed_at, created_at = prefix .. "record", prefix .. "indexed_at", prefix .. "created_at" + local created = record .. "::jsonb->>'createdAt'" + return "CASE WHEN jsonb_typeof(" .. record .. "::jsonb->'createdAt') = 'string' AND " .. created .. + " ~ '" .. CREATED_AT_SORT_PATTERN .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. + ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(" .. indexed_at .. + "::timestamptz, " .. created_at .. "::timestamptz) END" +end + local function valid_funding_record_uri(value) local valid, collection = valid_record_uri(value) if not valid or not valid_nsid(collection) then return false, collection end @@ -227,33 +289,11 @@ local function funding_receipt_array(key, format) return unique end -local valid_receipt_datetime = valid_datetime - local function decode_funding_receipt_cursor(token, direction) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end - end - local valid, collection = valid_record_uri(value.u) - if not valid_receipt_datetime(value.t) or not valid or collection ~= RECEIPT then - invalid("cursor is malformed") - end - return value -end - -local function encode_funding_receipt_cursor(value) - return (json.encode(value):gsub(".", function(char) - return string.format("%02x", string.byte(char)) - end)) + return decode_cursor(token, direction, function(value) + local valid, collection = valid_record_uri(value.u) + return valid_datetime(value.t) and valid and collection == RECEIPT + end) end local function bind_funding_receipt_values(values, items) @@ -266,10 +306,7 @@ local function bind_funding_receipt_values(values, items) end local function add_funding_receipt_array_filter(where, values, items, expression) - if items == nil then return end - if #items == 0 then where[#where + 1] = "FALSE"; return end - local marks = bind_funding_receipt_values(values, items) - where[#where + 1] = expression .. " IN (" .. table.concat(marks, ", ") .. ")" + if items ~= nil then add_in(where, values, expression, items) end end local function add_funding_receipt_party_filter(where, values, key, items) @@ -295,15 +332,6 @@ local function add_funding_receipt_party_filter(where, values, key, items) where[#where + 1] = "(" .. table.concat(matches, " OR ") .. ")" end -local function funding_receipt_sort_expression() - local created = "receipt.record::jsonb->>'createdAt'" - local zoned = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" - return "CASE WHEN jsonb_typeof(receipt.record::jsonb->'createdAt') = 'string' AND " .. created .. - " ~ '" .. zoned .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. - ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE " .. - "COALESCE(receipt.indexed_at::timestamptz, receipt.created_at::timestamptz) END" -end - local function query_funding_receipts(authors, uris, from_values, to_values, for_uris, transaction_ids, limit, cursor, direction) local where, values = { "receipt.collection = $1" }, { RECEIPT } add_funding_receipt_array_filter(where, values, authors, "receipt.did") @@ -329,7 +357,7 @@ local function query_funding_receipts(authors, uris, from_values, to_values, for "receipt.record::text AS record, to_char(sorted.sort_at AT TIME ZONE 'UTC', " .. "'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp " .. "FROM happyview_records AS receipt CROSS JOIN LATERAL (SELECT " .. - funding_receipt_sort_expression() .. " AS sort_at) AS sorted WHERE " .. table.concat(where, " AND ") .. + created_at_sort("receipt") .. " AS sort_at) AS sorted WHERE " .. table.concat(where, " AND ") .. " ORDER BY sorted.sort_at " .. ordering .. ", receipt.uri " .. ordering .. " LIMIT $" .. #values local rows = receipt_query(sql, values) local more = #rows > limit @@ -342,7 +370,7 @@ local function query_funding_receipts(authors, uris, from_values, to_values, for local next_cursor if more then local last = rows[#rows] - next_cursor = encode_funding_receipt_cursor({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) + next_cursor = cursor_encode({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) end return receipts, next_cursor end @@ -367,10 +395,7 @@ local function funding_receipts_response() local for_uris = funding_receipt_array("forUris", "atUri") local transaction_ids = funding_receipt_array("transactionIds") - local direction = scalar(params, "sortDirection") or "desc" - if direction ~= "asc" and direction ~= "desc" then - invalid("sortDirection must be 'asc' or 'desc'") - end + local direction = parse_sort_direction(params) local limit = parse_list_limit(params) local cursor = decode_funding_receipt_cursor(scalar(params, "cursor"), direction) local receipts, next_cursor = query_funding_receipts( diff --git a/happyview-modules/lua/endpoints/searchActivities.lua b/happyview-modules/lua/endpoints/searchActivities.lua index 63fbee71..ea055541 100644 --- a/happyview-modules/lua/endpoints/searchActivities.lua +++ b/happyview-modules/lua/endpoints/searchActivities.lua @@ -283,6 +283,53 @@ local function created_at_sort(alias) "::timestamptz, " .. created_at .. "::timestamptz) END" end +local function parse_sort_direction(params) + local direction = scalar(params, "sortDirection") or "desc" + if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + return direction +end + +local function cursor_encode(value) + local encoded = json.encode(value) + return (encoded:gsub(".", function(char) return string.format("%02x", string.byte(char)) end)) +end + +-- Decodes a hex-encoded {v, d, t, u} list cursor. Rejects malformed tokens, other +-- sort directions and unknown keys; validate_value(value) -> boolean runs the +-- endpoint-specific check on t and u. Not for endpoints that skip the 8192-character +-- cap or accept extra keys. +local function decode_cursor(token, direction, validate_value) + if token == nil then return nil end + if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then + invalid("cursor is malformed") + end + local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) + local ok, value = pcall(json.decode, decoded) + if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction + or type(value.t) ~= "string" or type(value.u) ~= "string" then + invalid("cursor is malformed or belongs to another sortDirection") + end + for key in pairs(value) do + if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end + end + if not validate_value(value) then invalid("cursor is malformed") end + return value +end + +-- Adds `column IN ($n, ...)` to where, binding items; an empty list matches nothing. +local function add_in(where, binds, column, items) + if #items == 0 then + where[#where + 1] = "FALSE" + return + end + local marks = {} + for _, item in ipairs(items) do + binds[#binds + 1] = item + marks[#marks + 1] = "$" .. #binds + end + where[#where + 1] = column .. " IN (" .. table.concat(marks, ", ") .. ")" +end + local function activity_scalar(values, key) if key == "hasOrganizationRecord" and type(values[key]) == "boolean" then return tostring(values[key]) @@ -333,30 +380,10 @@ local function activity_array(key, format) end local function decode_activity_cursor(token, direction) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end - end - local valid, collection = valid_record_uri(value.u) - if not valid_datetime(value.t) or not valid or collection ~= ACTIVITY then - invalid("cursor is malformed") - end - return value -end - -local function encode_activity_cursor(value) - return (json.encode(value):gsub(".", function(char) - return string.format("%02x", string.byte(char)) - end)) + return decode_cursor(token, direction, function(value) + local valid, collection = valid_record_uri(value.u) + return valid_datetime(value.t) and valid and collection == ACTIVITY + end) end local function bind_values(values, items) @@ -387,13 +414,7 @@ end local function activity_query(authors, has_organization_record, contributors, involved_actors, uris, search, limit, cursor, direction) local where, values = { "activity.collection = $1" }, { ACTIVITY } - if authors then - if #authors == 0 then - where[#where + 1] = "FALSE" - else - where[#where + 1] = "activity.did IN (" .. table.concat(bind_values(values, authors), ", ") .. ")" - end - end + if authors then add_in(where, values, "activity.did", authors) end if has_organization_record ~= nil then local predicate = has_organization_record and "EXISTS" or "NOT EXISTS" where[#where + 1] = predicate .. " (SELECT 1 FROM happyview_records AS organization " .. @@ -413,10 +434,7 @@ local function activity_query(authors, has_organization_record, contributors, in activity_contributor_match(values, involved_actors) .. ")" end end - if uris then - if #uris == 0 then where[#where + 1] = "FALSE" - else where[#where + 1] = "activity.uri IN (" .. table.concat(bind_values(values, uris), ", ") .. ")" end - end + if uris then add_in(where, values, "activity.uri", uris) end if search ~= nil then values[#values + 1] = search local pattern = "$" .. #values @@ -452,7 +470,7 @@ local function activity_query(authors, has_organization_record, contributors, in local next_cursor if more then local last = rows[#rows] - next_cursor = encode_activity_cursor({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) + next_cursor = cursor_encode({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) end return views, next_cursor end @@ -492,8 +510,7 @@ local function activity_list_response(search_enabled) search = search:match("^%s*(.-)%s*$") if search == "" then search = nil end end - local direction = scalar(params, "sortDirection") or "desc" - if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + local direction = parse_sort_direction(params) local limit = parse_list_limit(params) local cursor = decode_activity_cursor(scalar(params, "cursor"), direction) local activities, next_cursor = activity_query( diff --git a/happyview-modules/lua/endpoints/searchBadgeDefinitions.lua b/happyview-modules/lua/endpoints/searchBadgeDefinitions.lua index e78a63ea..a0e4223c 100644 --- a/happyview-modules/lua/endpoints/searchBadgeDefinitions.lua +++ b/happyview-modules/lua/endpoints/searchBadgeDefinitions.lua @@ -39,6 +39,13 @@ local function valid_record_uri(value) return true, collection, authority end +local BADGE_DEFINITION_COLLECTION = "app.certified.badge.definition" + +local function valid_badge_definition_uri(value) + local valid, collection = valid_record_uri(value) + return valid and collection == BADGE_DEFINITION_COLLECTION +end + local function valid_datetime(value) if type(value) ~= "string" then return false end local year, month, day, hour, minute, second, suffix = value:match( @@ -126,11 +133,6 @@ end local COLLECTION = "app.certified.badge.definition" -local function valid_badge_definition_uri(value) - local valid, collection = valid_record_uri(value) - return valid and collection == COLLECTION -end - local function query(sql, values) if db.backend() ~= "postgres" then error("BadgeDefinitionQueryFailed: badge definition API requires PostgreSQL", 0) diff --git a/happyview-modules/lua/endpoints/searchCollections.lua b/happyview-modules/lua/endpoints/searchCollections.lua index 553705b3..16b79369 100644 --- a/happyview-modules/lua/endpoints/searchCollections.lua +++ b/happyview-modules/lua/endpoints/searchCollections.lua @@ -291,6 +291,82 @@ local function created_at_sort(alias) "::timestamptz, " .. created_at .. "::timestamptz) END" end +local function invalid(message) + error("InvalidRequest: " .. message, 0) +end + +local function keys_only(values, allowed, unknown_message_prefix) + for key in pairs(values) do + if not allowed[key] then + invalid(unknown_message_prefix and (unknown_message_prefix .. key) or "unknown query parameter") + end + end +end + +local function scalar(params, key) + local value = params[key] + if value == nil then return nil end + if type(value) ~= "string" and type(value) ~= "number" then + invalid(key .. " must occur once") + end + return tostring(value) +end + +local function parse_list_limit(params) + local limit_value = scalar(params, "limit") + if limit_value and not limit_value:match("^%d+$") then invalid("limit must be an integer from 1 through 100") end + local limit = limit_value and tonumber(limit_value) or 25 + if not limit or limit % 1 ~= 0 or limit < 1 or limit > 100 then invalid("limit must be an integer from 1 through 100") end + return limit +end + +local function parse_sort_direction(params) + local direction = scalar(params, "sortDirection") or "desc" + if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + return direction +end + +local function cursor_encode(value) + local encoded = json.encode(value) + return (encoded:gsub(".", function(char) return string.format("%02x", string.byte(char)) end)) +end + +-- Decodes a hex-encoded {v, d, t, u} list cursor. Rejects malformed tokens, other +-- sort directions and unknown keys; validate_value(value) -> boolean runs the +-- endpoint-specific check on t and u. Not for endpoints that skip the 8192-character +-- cap or accept extra keys. +local function decode_cursor(token, direction, validate_value) + if token == nil then return nil end + if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then + invalid("cursor is malformed") + end + local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) + local ok, value = pcall(json.decode, decoded) + if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction + or type(value.t) ~= "string" or type(value.u) ~= "string" then + invalid("cursor is malformed or belongs to another sortDirection") + end + for key in pairs(value) do + if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end + end + if not validate_value(value) then invalid("cursor is malformed") end + return value +end + +-- Adds `column IN ($n, ...)` to where, binding items; an empty list matches nothing. +local function add_in(where, binds, column, items) + if #items == 0 then + where[#where + 1] = "FALSE" + return + end + local marks = {} + for _, item in ipairs(items) do + binds[#binds + 1] = item + marks[#marks + 1] = "$" .. #binds + end + where[#where + 1] = column .. " IN (" .. table.concat(marks, ", ") .. ")" +end + local function collection_array(key, kind) local value = params[key] if value == nil then return nil end @@ -345,44 +421,11 @@ local function collection_array(key, kind) return unique end -local function collection_list_limit() - local value = collection_scalar(params, "limit") - if value == nil then return 25 end - if not value:match("^%d+$") then collection_invalid("limit must be an integer from 1 through 100") end - local limit = tonumber(value) - if not limit or limit < 1 or limit > 100 then - collection_invalid("limit must be an integer from 1 through 100") - end - return limit -end - local function collection_list_cursor_decode(token, direction) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - collection_invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - collection_invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then - collection_invalid("cursor is malformed") - end - end - local valid, collection = collection_valid_record_uri(value.u) - if not valid_datetime(value.t) or not valid or collection ~= COLLECTION then - collection_invalid("cursor is malformed") - end - return value -end - -local function collection_cursor_encode(value) - return (json.encode(value):gsub(".", function(char) - return string.format("%02x", string.byte(char)) - end)) + return decode_cursor(token, direction, function(value) + local valid, collection = collection_valid_record_uri(value.u) + return valid_datetime(value.t) and valid and collection == COLLECTION + end) end local function collection_bind_values(values, items) @@ -397,34 +440,15 @@ end local function collection_list_filters(filters, search) local where, values = { "collection.collection = $1" }, { COLLECTION } - if filters.authors then - if #filters.authors == 0 then - where[#where + 1] = "FALSE" - else - where[#where + 1] = "collection.did IN (" .. table.concat(collection_bind_values(values, filters.authors), ", ") .. ")" - end - end + if filters.authors then add_in(where, values, "collection.did", filters.authors) end if filters.hasOrganizationRecord ~= nil then local predicate = filters.hasOrganizationRecord and "EXISTS" or "NOT EXISTS" where[#where + 1] = predicate .. " (SELECT 1 FROM happyview_records AS organization " .. "WHERE organization.collection = 'app.certified.actor.organization' AND organization.rkey = 'self' " .. "AND organization.did = collection.did)" end - if filters.types then - if #filters.types == 0 then - where[#where + 1] = "FALSE" - else - where[#where + 1] = "collection.record::jsonb->>'type' IN (" .. - table.concat(collection_bind_values(values, filters.types), ", ") .. ")" - end - end - if filters.uris then - if #filters.uris == 0 then - where[#where + 1] = "FALSE" - else - where[#where + 1] = "collection.uri IN (" .. table.concat(collection_bind_values(values, filters.uris), ", ") .. ")" - end - end + if filters.types then add_in(where, values, "collection.record::jsonb->>'type'", filters.types) end + if filters.uris then add_in(where, values, "collection.uri", filters.uris) end if filters.itemUris then if #filters.itemUris == 0 then where[#where + 1] = "FALSE" @@ -491,7 +515,7 @@ local function collection_list_page(rows, limit, direction) local next_cursor if more then local last = rows[#rows] - next_cursor = collection_cursor_encode({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) + next_cursor = cursor_encode({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) end return views, next_cursor end @@ -540,11 +564,8 @@ local function collection_list_response(search_enabled) search = search:match("^%s*(.-)%s*$") if search == "" then search = nil end end - local direction = collection_scalar(params, "sortDirection") or "desc" - if direction ~= "asc" and direction ~= "desc" then - collection_invalid("sortDirection must be 'asc' or 'desc'") - end - local limit = collection_list_limit() + local direction = parse_sort_direction(params) + local limit = parse_list_limit(params) local cursor = collection_list_cursor_decode(collection_scalar(params, "cursor"), direction) local collections, next_cursor = collection_list_query({ authors = authors, diff --git a/happyview-modules/lua/shared/activityList.lua b/happyview-modules/lua/shared/activityList.lua index 603855c0..9bcef39a 100644 --- a/happyview-modules/lua/shared/activityList.lua +++ b/happyview-modules/lua/shared/activityList.lua @@ -48,30 +48,10 @@ local function activity_array(key, format) end local function decode_activity_cursor(token, direction) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end - end - local valid, collection = valid_record_uri(value.u) - if not valid_datetime(value.t) or not valid or collection ~= ACTIVITY then - invalid("cursor is malformed") - end - return value -end - -local function encode_activity_cursor(value) - return (json.encode(value):gsub(".", function(char) - return string.format("%02x", string.byte(char)) - end)) + return decode_cursor(token, direction, function(value) + local valid, collection = valid_record_uri(value.u) + return valid_datetime(value.t) and valid and collection == ACTIVITY + end) end local function bind_values(values, items) @@ -102,13 +82,7 @@ end local function activity_query(authors, has_organization_record, contributors, involved_actors, uris, search, limit, cursor, direction) local where, values = { "activity.collection = $1" }, { ACTIVITY } - if authors then - if #authors == 0 then - where[#where + 1] = "FALSE" - else - where[#where + 1] = "activity.did IN (" .. table.concat(bind_values(values, authors), ", ") .. ")" - end - end + if authors then add_in(where, values, "activity.did", authors) end if has_organization_record ~= nil then local predicate = has_organization_record and "EXISTS" or "NOT EXISTS" where[#where + 1] = predicate .. " (SELECT 1 FROM happyview_records AS organization " .. @@ -128,10 +102,7 @@ local function activity_query(authors, has_organization_record, contributors, in activity_contributor_match(values, involved_actors) .. ")" end end - if uris then - if #uris == 0 then where[#where + 1] = "FALSE" - else where[#where + 1] = "activity.uri IN (" .. table.concat(bind_values(values, uris), ", ") .. ")" end - end + if uris then add_in(where, values, "activity.uri", uris) end if search ~= nil then values[#values + 1] = search local pattern = "$" .. #values @@ -167,7 +138,7 @@ local function activity_query(authors, has_organization_record, contributors, in local next_cursor if more then local last = rows[#rows] - next_cursor = encode_activity_cursor({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) + next_cursor = cursor_encode({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) end return views, next_cursor end @@ -207,8 +178,7 @@ local function activity_list_response(search_enabled) search = search:match("^%s*(.-)%s*$") if search == "" then search = nil end end - local direction = scalar(params, "sortDirection") or "desc" - if direction ~= "asc" and direction ~= "desc" then invalid("sortDirection must be 'asc' or 'desc'") end + local direction = parse_sort_direction(params) local limit = parse_list_limit(params) local cursor = decode_activity_cursor(scalar(params, "cursor"), direction) local activities, next_cursor = activity_query( diff --git a/happyview-modules/lua/shared/attachmentList.lua b/happyview-modules/lua/shared/attachmentList.lua index 5797a309..aaaa6145 100644 --- a/happyview-modules/lua/shared/attachmentList.lua +++ b/happyview-modules/lua/shared/attachmentList.lua @@ -33,20 +33,6 @@ local function attachment_array(params, key, validate, description) return unique end -local function attachment_bind_in(where, values, expression, items) - if not items then return end - if #items == 0 then - where[#where + 1] = "FALSE" - return - end - local placeholders = {} - for _, item in ipairs(items) do - values[#values + 1] = item - placeholders[#placeholders + 1] = "$" .. #values - end - where[#where + 1] = expression .. " IN (" .. table.concat(placeholders, ", ") .. ")" -end - local function attachment_subject_filter(where, values, subjects) if not subjects then return end if #subjects == 0 then @@ -64,39 +50,20 @@ local function attachment_subject_filter(where, values, subjects) "WHERE subject.value->>'uri' IN (" .. table.concat(placeholders, ", ") .. "))" end -local function attachment_sort_expression() - local created = "attachment.record::jsonb->>'createdAt'" - local zoned = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" - return "CASE WHEN jsonb_typeof(attachment.record::jsonb->'createdAt') = 'string' AND " .. created .. - " ~ '" .. zoned .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. - ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE " .. - "COALESCE(attachment.indexed_at::timestamptz, attachment.created_at::timestamptz) END" -end - local function attachment_cursor_decode(token, direction) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end - end - if not valid_datetime(value.t) or not valid_attachment_uri(value.u) then invalid("cursor is malformed") end - return value + return decode_cursor(token, direction, function(value) + return valid_datetime(value.t) and valid_attachment_uri(value.u) + end) end local function query_attachments(filters, limit, cursor, direction) local where, values = { "attachment.collection = $1" }, { ATTACHMENT_COLLECTION } - attachment_bind_in(where, values, "attachment.did", filters.authors) - attachment_bind_in(where, values, "attachment.uri", filters.uris) + if filters.authors then add_in(where, values, "attachment.did", filters.authors) end + if filters.uris then add_in(where, values, "attachment.uri", filters.uris) end attachment_subject_filter(where, values, filters.subjects) - attachment_bind_in(where, values, "attachment.record::jsonb->>'contentType'", filters.content_types) + if filters.content_types then + add_in(where, values, "attachment.record::jsonb->>'contentType'", filters.content_types) + end if cursor then values[#values + 1] = cursor.t @@ -110,7 +77,7 @@ local function query_attachments(filters, limit, cursor, direction) values[#values + 1] = limit + 1 local ordering = direction == "asc" and "ASC" or "DESC" - local sort_key = attachment_sort_expression() + local sort_key = created_at_sort("attachment") local sql = "SELECT attachment.uri, attachment.did, attachment.cid, attachment.indexed_at::text AS indexed_at, " .. "attachment.record::text AS record, to_char(sorted.sort_at AT TIME ZONE 'UTC', " .. "'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp " .. diff --git a/happyview-modules/lua/shared/collectionItems.lua b/happyview-modules/lua/shared/collectionItems.lua index 6794b938..6375399e 100644 --- a/happyview-modules/lua/shared/collectionItems.lua +++ b/happyview-modules/lua/shared/collectionItems.lua @@ -11,17 +11,6 @@ local function collection_items_keys_only(values, allowed) keys_only(values, allowed, "unknown query parameter: ") end -local function collection_items_limit() - local value = scalar(params, "limit") - if value == nil then return 25 end - if not value:match("^%d+$") then collection_items_invalid("limit must be an integer from 1 through 100") end - local limit = tonumber(value) - if not limit or limit < 1 or limit > 100 then - collection_items_invalid("limit must be an integer from 1 through 100") - end - return limit -end - local function collection_items_cursor_decode(token, collection_uri) if token == nil then return nil end if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then @@ -165,7 +154,7 @@ local function collection_items_response() if not collection_uri or not valid or collection ~= ITEM_COLLECTION then collection_items_invalid("collection must be a full org.hypercerts.collection AT-URI with a DID authority") end - local limit = collection_items_limit() + local limit = parse_list_limit(params) local cursor = collection_items_cursor_decode(scalar(params, "cursor"), collection_uri) local collection_rows = collection_items_query( diff --git a/happyview-modules/lua/shared/collectionList.lua b/happyview-modules/lua/shared/collectionList.lua index de89183c..8520fc91 100644 --- a/happyview-modules/lua/shared/collectionList.lua +++ b/happyview-modules/lua/shared/collectionList.lua @@ -52,44 +52,11 @@ local function collection_array(key, kind) return unique end -local function collection_list_limit() - local value = collection_scalar(params, "limit") - if value == nil then return 25 end - if not value:match("^%d+$") then collection_invalid("limit must be an integer from 1 through 100") end - local limit = tonumber(value) - if not limit or limit < 1 or limit > 100 then - collection_invalid("limit must be an integer from 1 through 100") - end - return limit -end - local function collection_list_cursor_decode(token, direction) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - collection_invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - collection_invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then - collection_invalid("cursor is malformed") - end - end - local valid, collection = collection_valid_record_uri(value.u) - if not valid_datetime(value.t) or not valid or collection ~= COLLECTION then - collection_invalid("cursor is malformed") - end - return value -end - -local function collection_cursor_encode(value) - return (json.encode(value):gsub(".", function(char) - return string.format("%02x", string.byte(char)) - end)) + return decode_cursor(token, direction, function(value) + local valid, collection = collection_valid_record_uri(value.u) + return valid_datetime(value.t) and valid and collection == COLLECTION + end) end local function collection_bind_values(values, items) @@ -104,34 +71,15 @@ end local function collection_list_filters(filters, search) local where, values = { "collection.collection = $1" }, { COLLECTION } - if filters.authors then - if #filters.authors == 0 then - where[#where + 1] = "FALSE" - else - where[#where + 1] = "collection.did IN (" .. table.concat(collection_bind_values(values, filters.authors), ", ") .. ")" - end - end + if filters.authors then add_in(where, values, "collection.did", filters.authors) end if filters.hasOrganizationRecord ~= nil then local predicate = filters.hasOrganizationRecord and "EXISTS" or "NOT EXISTS" where[#where + 1] = predicate .. " (SELECT 1 FROM happyview_records AS organization " .. "WHERE organization.collection = 'app.certified.actor.organization' AND organization.rkey = 'self' " .. "AND organization.did = collection.did)" end - if filters.types then - if #filters.types == 0 then - where[#where + 1] = "FALSE" - else - where[#where + 1] = "collection.record::jsonb->>'type' IN (" .. - table.concat(collection_bind_values(values, filters.types), ", ") .. ")" - end - end - if filters.uris then - if #filters.uris == 0 then - where[#where + 1] = "FALSE" - else - where[#where + 1] = "collection.uri IN (" .. table.concat(collection_bind_values(values, filters.uris), ", ") .. ")" - end - end + if filters.types then add_in(where, values, "collection.record::jsonb->>'type'", filters.types) end + if filters.uris then add_in(where, values, "collection.uri", filters.uris) end if filters.itemUris then if #filters.itemUris == 0 then where[#where + 1] = "FALSE" @@ -198,7 +146,7 @@ local function collection_list_page(rows, limit, direction) local next_cursor if more then local last = rows[#rows] - next_cursor = collection_cursor_encode({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) + next_cursor = cursor_encode({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) end return views, next_cursor end @@ -247,11 +195,8 @@ local function collection_list_response(search_enabled) search = search:match("^%s*(.-)%s*$") if search == "" then search = nil end end - local direction = collection_scalar(params, "sortDirection") or "desc" - if direction ~= "asc" and direction ~= "desc" then - collection_invalid("sortDirection must be 'asc' or 'desc'") - end - local limit = collection_list_limit() + local direction = parse_sort_direction(params) + local limit = parse_list_limit(params) local cursor = collection_list_cursor_decode(collection_scalar(params, "cursor"), direction) local collections, next_cursor = collection_list_query({ authors = authors, diff --git a/happyview-modules/lua/shared/entityFollow.lua b/happyview-modules/lua/shared/entityFollow.lua index 2b8c346b..64468362 100644 --- a/happyview-modules/lua/shared/entityFollow.lua +++ b/happyview-modules/lua/shared/entityFollow.lua @@ -12,13 +12,6 @@ local function entity_follow_query(sql, values) return result end --- Keep malformed publisher timestamps out of casts; preserve the actor-follow fallback order. -local function entity_follow_sort_key() - local created = "record::jsonb->>'createdAt'" - local zoned = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" - return "CASE WHEN jsonb_typeof(record::jsonb->'createdAt') = 'string' AND " .. created .. " ~ '" .. zoned .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(indexed_at::timestamptz, created_at::timestamptz) END" -end - local function entity_follow_indexed_record_view(row) return { uri = row.uri, diff --git a/happyview-modules/lua/shared/entityFollowLookup.lua b/happyview-modules/lua/shared/entityFollowLookup.lua index f89c0e09..b811b7c1 100644 --- a/happyview-modules/lua/shared/entityFollowLookup.lua +++ b/happyview-modules/lua/shared/entityFollowLookup.lua @@ -1,6 +1,6 @@ local function entity_follow_lookup(actor, entity_uri) local rows = entity_follow_query( - "SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records CROSS JOIN LATERAL (SELECT " .. entity_follow_sort_key() .. " AS sort_at) sorted WHERE collection = $1 AND did = $2 AND record::jsonb->'subject'->>'uri' = $3 ORDER BY sorted.sort_at ASC, uri ASC LIMIT 1", + "SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record FROM happyview_records CROSS JOIN LATERAL (SELECT " .. created_at_sort() .. " AS sort_at) sorted WHERE collection = $1 AND did = $2 AND record::jsonb->'subject'->>'uri' = $3 ORDER BY sorted.sort_at ASC, uri ASC LIMIT 1", { ENTITY_FOLLOW, actor, entity_uri }) if #rows == 0 then return ENTITY_FOLLOW_NULL end return entity_follow_record_view(rows[1]) diff --git a/happyview-modules/lua/shared/entityFollowPagination.lua b/happyview-modules/lua/shared/entityFollowPagination.lua index ecafb5cb..84891341 100644 --- a/happyview-modules/lua/shared/entityFollowPagination.lua +++ b/happyview-modules/lua/shared/entityFollowPagination.lua @@ -1,22 +1,8 @@ local function entity_follow_decode_cursor(token, direction) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end - end - local valid, collection = valid_record_uri(value.u) - if not valid or collection ~= ENTITY_FOLLOW or not valid_datetime(value.t) then - invalid("cursor is malformed") - end - return value + return decode_cursor(token, direction, function(value) + local valid, collection = valid_record_uri(value.u) + return valid and collection == ENTITY_FOLLOW and valid_datetime(value.t) + end) end local function entity_follow_query_page(mode, identity, limit, cursor, direction) @@ -40,7 +26,7 @@ local function entity_follow_query_page(mode, identity, limit, cursor, direction values[#values + 1] = limit + 1 local ordering = direction == "asc" and "ASC" or "DESC" local subject_uri = "record::jsonb->'subject'->>'uri'" - local ranked_follows = "ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, " .. subject_uri .. " ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. entity_follow_sort_key() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. ")" + local ranked_follows = "ranked_follows AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, " .. subject_uri .. " ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. created_at_sort() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. ")" local representatives = "representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at FROM ranked_follows WHERE relationship_rank = 1)" local page = "page AS (SELECT uri, did, cid, indexed_at, record, sort_at FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values .. ")" local sql diff --git a/happyview-modules/lua/shared/evaluationList.lua b/happyview-modules/lua/shared/evaluationList.lua index febddba5..e9ecbc78 100644 --- a/happyview-modules/lua/shared/evaluationList.lua +++ b/happyview-modules/lua/shared/evaluationList.lua @@ -58,46 +58,16 @@ local function evaluation_evaluator_match(values, dids) "WHERE evaluator.value->>'did' IN (" .. table.concat(marks, ", ") .. "))" end -local function evaluation_sort_expression() - local created = "evaluation.record::jsonb->>'createdAt'" - local zoned = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" - return "CASE WHEN jsonb_typeof(evaluation.record::jsonb->'createdAt') = 'string' AND " .. created .. - " ~ '" .. zoned .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. - ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE " .. - "COALESCE(evaluation.indexed_at::timestamptz, evaluation.created_at::timestamptz) END" -end - local function decode_evaluation_cursor(token, direction) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end - end - local valid, collection = valid_record_uri(value.u) - if not valid_datetime(value.t) or not valid or collection ~= EVALUATION then - invalid("cursor is malformed") - end - return value + return decode_cursor(token, direction, function(value) + local valid, collection = valid_record_uri(value.u) + return valid_datetime(value.t) and valid and collection == EVALUATION + end) end local function evaluation_list_filters(authors, evaluators, subjects) local where, values = { "evaluation.collection = $1" }, { EVALUATION } - if authors then - if #authors == 0 then - where[#where + 1] = "FALSE" - else - where[#where + 1] = "evaluation.did IN (" .. - table.concat(bind_evaluation_values(values, authors), ", ") .. ")" - end - end + if authors then add_in(where, values, "evaluation.did", authors) end if evaluators then if #evaluators == 0 then where[#where + 1] = "FALSE" @@ -105,14 +75,7 @@ local function evaluation_list_filters(authors, evaluators, subjects) where[#where + 1] = evaluation_evaluator_match(values, evaluators) end end - if subjects then - if #subjects == 0 then - where[#where + 1] = "FALSE" - else - where[#where + 1] = "evaluation.record::jsonb->'subject'->>'uri' IN (" .. - table.concat(bind_evaluation_values(values, subjects), ", ") .. ")" - end - end + if subjects then add_in(where, values, "evaluation.record::jsonb->'subject'->>'uri'", subjects) end return where, values end @@ -137,7 +100,7 @@ local function evaluation_list_page_query(values, where, cursor, limit, directio "to_char(sorted.sort_at AT TIME ZONE 'UTC', " .. "'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp " .. "FROM happyview_records AS evaluation CROSS JOIN LATERAL (SELECT " .. - evaluation_sort_expression() .. " AS sort_at) AS sorted WHERE " .. table.concat(page_where, " AND ") .. + created_at_sort("evaluation") .. " AS sort_at) AS sorted WHERE " .. table.concat(page_where, " AND ") .. " ORDER BY sorted.sort_at " .. ordering .. ", evaluation.uri " .. ordering .. " LIMIT $" .. #page_values return sql, page_values end diff --git a/happyview-modules/lua/shared/feedReaction.lua b/happyview-modules/lua/shared/feedReaction.lua index fa2f694c..28215b60 100644 --- a/happyview-modules/lua/shared/feedReaction.lua +++ b/happyview-modules/lua/shared/feedReaction.lua @@ -13,12 +13,6 @@ local FEED_REACTIONS = { }, } -local function feed_reaction_sort_key() - local created = "record::jsonb->>'createdAt'" - local zoned = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" - return "CASE WHEN jsonb_typeof(record::jsonb->'createdAt') = 'string' AND " .. created .. " ~ '" .. zoned .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(indexed_at::timestamptz, created_at::timestamptz) END" -end - local function feed_reaction_query(reaction, sql, values) if db.backend() ~= "postgres" then error(reaction.failure .. ": reaction queries require PostgreSQL", 0) end local ok, result = pcall(db.raw, sql, values) @@ -27,24 +21,10 @@ local function feed_reaction_query(reaction, sql, values) end local function feed_reaction_decode_cursor(token, direction, collection) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end - end - local valid, cursor_collection = valid_record_uri(value.u) - if not valid or cursor_collection ~= collection or not valid_datetime(value.t) then - invalid("cursor is malformed") - end - return value + return decode_cursor(token, direction, function(value) + local valid, cursor_collection = valid_record_uri(value.u) + return valid and cursor_collection == collection and valid_datetime(value.t) + end) end local function feed_reaction_query_page(reaction, mode, identity, limit, cursor, direction) @@ -70,7 +50,7 @@ local function feed_reaction_query_page(reaction, mode, identity, limit, cursor, local ordering = direction == "asc" and "ASC" or "DESC" local subject_uri = "record::jsonb->'subject'->>'uri'" -- Select the earliest representative before applying the caller's independent page direction. - local sql = "WITH ranked_reactions AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, " .. subject_uri .. " ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. feed_reaction_sort_key() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. "), " .. + local sql = "WITH ranked_reactions AS (SELECT uri, did, cid, indexed_at::text AS indexed_at, record::text AS record, sorted.sort_at, ROW_NUMBER() OVER (PARTITION BY did, " .. subject_uri .. " ORDER BY sorted.sort_at ASC, uri ASC) AS relationship_rank FROM happyview_records CROSS JOIN LATERAL (SELECT " .. created_at_sort() .. " AS sort_at) sorted WHERE " .. table.concat(filters, " AND ") .. "), " .. "representatives AS (SELECT uri, did, cid, indexed_at, record, sort_at FROM ranked_reactions WHERE relationship_rank = 1), " local page = "page AS (SELECT uri, did, cid, indexed_at, record, sort_at FROM representatives" .. cursor_filter .. " ORDER BY sort_at " .. ordering .. ", uri " .. ordering .. " LIMIT $" .. #values .. ") " diff --git a/happyview-modules/lua/shared/fundingReceiptList.lua b/happyview-modules/lua/shared/fundingReceiptList.lua index 263b9655..468ba64e 100644 --- a/happyview-modules/lua/shared/fundingReceiptList.lua +++ b/happyview-modules/lua/shared/fundingReceiptList.lua @@ -60,33 +60,11 @@ local function funding_receipt_array(key, format) return unique end -local valid_receipt_datetime = valid_datetime - local function decode_funding_receipt_cursor(token, direction) - if token == nil then return nil end - if #token == 0 or #token > 8192 or #token % 2 ~= 0 or token:find("[^0-9a-f]") then - invalid("cursor is malformed") - end - local decoded = token:gsub("..", function(pair) return string.char(tonumber(pair, 16)) end) - local ok, value = pcall(json.decode, decoded) - if not ok or type(value) ~= "table" or value.v ~= 1 or value.d ~= direction - or type(value.t) ~= "string" or type(value.u) ~= "string" then - invalid("cursor is malformed or belongs to another sortDirection") - end - for key in pairs(value) do - if key ~= "v" and key ~= "d" and key ~= "t" and key ~= "u" then invalid("cursor is malformed") end - end - local valid, collection = valid_record_uri(value.u) - if not valid_receipt_datetime(value.t) or not valid or collection ~= RECEIPT then - invalid("cursor is malformed") - end - return value -end - -local function encode_funding_receipt_cursor(value) - return (json.encode(value):gsub(".", function(char) - return string.format("%02x", string.byte(char)) - end)) + return decode_cursor(token, direction, function(value) + local valid, collection = valid_record_uri(value.u) + return valid_datetime(value.t) and valid and collection == RECEIPT + end) end local function bind_funding_receipt_values(values, items) @@ -99,10 +77,7 @@ local function bind_funding_receipt_values(values, items) end local function add_funding_receipt_array_filter(where, values, items, expression) - if items == nil then return end - if #items == 0 then where[#where + 1] = "FALSE"; return end - local marks = bind_funding_receipt_values(values, items) - where[#where + 1] = expression .. " IN (" .. table.concat(marks, ", ") .. ")" + if items ~= nil then add_in(where, values, expression, items) end end local function add_funding_receipt_party_filter(where, values, key, items) @@ -128,15 +103,6 @@ local function add_funding_receipt_party_filter(where, values, key, items) where[#where + 1] = "(" .. table.concat(matches, " OR ") .. ")" end -local function funding_receipt_sort_expression() - local created = "receipt.record::jsonb->>'createdAt'" - local zoned = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" - return "CASE WHEN jsonb_typeof(receipt.record::jsonb->'createdAt') = 'string' AND " .. created .. - " ~ '" .. zoned .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. - ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE " .. - "COALESCE(receipt.indexed_at::timestamptz, receipt.created_at::timestamptz) END" -end - local function query_funding_receipts(authors, uris, from_values, to_values, for_uris, transaction_ids, limit, cursor, direction) local where, values = { "receipt.collection = $1" }, { RECEIPT } add_funding_receipt_array_filter(where, values, authors, "receipt.did") @@ -162,7 +128,7 @@ local function query_funding_receipts(authors, uris, from_values, to_values, for "receipt.record::text AS record, to_char(sorted.sort_at AT TIME ZONE 'UTC', " .. "'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS sort_timestamp " .. "FROM happyview_records AS receipt CROSS JOIN LATERAL (SELECT " .. - funding_receipt_sort_expression() .. " AS sort_at) AS sorted WHERE " .. table.concat(where, " AND ") .. + created_at_sort("receipt") .. " AS sort_at) AS sorted WHERE " .. table.concat(where, " AND ") .. " ORDER BY sorted.sort_at " .. ordering .. ", receipt.uri " .. ordering .. " LIMIT $" .. #values local rows = receipt_query(sql, values) local more = #rows > limit @@ -175,7 +141,7 @@ local function query_funding_receipts(authors, uris, from_values, to_values, for local next_cursor if more then local last = rows[#rows] - next_cursor = encode_funding_receipt_cursor({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) + next_cursor = cursor_encode({ v = 1, d = direction, t = last.sort_timestamp, u = last.uri }) end return receipts, next_cursor end @@ -200,10 +166,7 @@ local function funding_receipts_response() local for_uris = funding_receipt_array("forUris", "atUri") local transaction_ids = funding_receipt_array("transactionIds") - local direction = scalar(params, "sortDirection") or "desc" - if direction ~= "asc" and direction ~= "desc" then - invalid("sortDirection must be 'asc' or 'desc'") - end + local direction = parse_sort_direction(params) local limit = parse_list_limit(params) local cursor = decode_funding_receipt_cursor(scalar(params, "cursor"), direction) local receipts, next_cursor = query_funding_receipts( diff --git a/happyview-modules/lua/src/searchBadgeDefinitions.lua b/happyview-modules/lua/src/searchBadgeDefinitions.lua index 0fe5b7df..32706d6c 100644 --- a/happyview-modules/lua/src/searchBadgeDefinitions.lua +++ b/happyview-modules/lua/src/searchBadgeDefinitions.lua @@ -1,10 +1,5 @@ local COLLECTION = "app.certified.badge.definition" -local function valid_badge_definition_uri(value) - local valid, collection = valid_record_uri(value) - return valid and collection == COLLECTION -end - local function query(sql, values) if db.backend() ~= "postgres" then error("BadgeDefinitionQueryFailed: badge definition API requires PostgreSQL", 0) diff --git a/happyview-modules/modules/activity/manifest.json b/happyview-modules/modules/activity/manifest.json index 1344ac7d..95600686 100644 --- a/happyview-modules/modules/activity/manifest.json +++ b/happyview-modules/modules/activity/manifest.json @@ -52,6 +52,9 @@ "../../lua/shared/datetimeValidation.lua", "../../lua/shared/listValidation.lua", "../../lua/shared/createdAtSort.lua", + "../../lua/shared/listQuery.lua", + "../../lua/shared/listCursor.lua", + "../../lua/shared/listWhere.lua", "../../lua/shared/activityList.lua" ], "config": { @@ -74,6 +77,9 @@ "../../lua/shared/datetimeValidation.lua", "../../lua/shared/listValidation.lua", "../../lua/shared/createdAtSort.lua", + "../../lua/shared/listQuery.lua", + "../../lua/shared/listCursor.lua", + "../../lua/shared/listWhere.lua", "../../lua/shared/activityList.lua" ], "config": { diff --git a/happyview-modules/modules/badge-queries/manifest.json b/happyview-modules/modules/badge-queries/manifest.json index 04b597b8..0d4b8f17 100644 --- a/happyview-modules/modules/badge-queries/manifest.json +++ b/happyview-modules/modules/badge-queries/manifest.json @@ -64,6 +64,7 @@ "../../lua/shared/query.lua", "../../lua/shared/didValidation.lua", "../../lua/shared/recordIdentifier.lua", + "../../lua/shared/badgeDefinitionValidation.lua", "../../lua/shared/datetimeValidation.lua", "../../lua/shared/listValidation.lua", "../../lua/shared/listQuery.lua", diff --git a/happyview-modules/modules/collection/manifest.json b/happyview-modules/modules/collection/manifest.json index 39d3e11f..caa471b3 100644 --- a/happyview-modules/modules/collection/manifest.json +++ b/happyview-modules/modules/collection/manifest.json @@ -39,6 +39,11 @@ "../../lua/shared/collection.lua", "../../lua/shared/datetimeValidation.lua", "../../lua/shared/createdAtSort.lua", + "../../lua/shared/query.lua", + "../../lua/shared/listValidation.lua", + "../../lua/shared/listQuery.lua", + "../../lua/shared/listCursor.lua", + "../../lua/shared/listWhere.lua", "../../lua/shared/collectionList.lua" ], "config": { @@ -69,6 +74,11 @@ "../../lua/shared/collection.lua", "../../lua/shared/datetimeValidation.lua", "../../lua/shared/createdAtSort.lua", + "../../lua/shared/query.lua", + "../../lua/shared/listValidation.lua", + "../../lua/shared/listQuery.lua", + "../../lua/shared/listCursor.lua", + "../../lua/shared/listWhere.lua", "../../lua/shared/collectionList.lua" ], "config": { @@ -88,6 +98,7 @@ "../../lua/shared/recordIdentifier.lua", "../../lua/shared/activityProjection.lua", "../../lua/shared/featureProjection.lua", + "../../lua/shared/listValidation.lua", "../../lua/shared/collectionItems.lua" ], "config": { diff --git a/happyview-modules/modules/context-attachments/manifest.json b/happyview-modules/modules/context-attachments/manifest.json index 55158f9d..047e6233 100644 --- a/happyview-modules/modules/context-attachments/manifest.json +++ b/happyview-modules/modules/context-attachments/manifest.json @@ -54,6 +54,9 @@ "../../lua/shared/recordView.lua", "../../lua/shared/actorView.lua", "../../lua/shared/attachment.lua", + "../../lua/shared/listCursor.lua", + "../../lua/shared/listWhere.lua", + "../../lua/shared/createdAtSort.lua", "../../lua/shared/attachmentList.lua" ], "config": { diff --git a/happyview-modules/modules/context-evaluation/manifest.json b/happyview-modules/modules/context-evaluation/manifest.json index e0b0c434..4268ea60 100644 --- a/happyview-modules/modules/context-evaluation/manifest.json +++ b/happyview-modules/modules/context-evaluation/manifest.json @@ -55,6 +55,9 @@ "../../lua/shared/actorView.lua", "../../lua/shared/evaluation.lua", "../../lua/shared/nsidValidation.lua", + "../../lua/shared/listCursor.lua", + "../../lua/shared/listWhere.lua", + "../../lua/shared/createdAtSort.lua", "../../lua/shared/evaluationList.lua" ], "config": { diff --git a/happyview-modules/modules/entity-follow/manifest.json b/happyview-modules/modules/entity-follow/manifest.json index 00045350..69ad1b3f 100644 --- a/happyview-modules/modules/entity-follow/manifest.json +++ b/happyview-modules/modules/entity-follow/manifest.json @@ -55,6 +55,7 @@ "../../lua/shared/query.lua", "../../lua/shared/recordIdentifier.lua", "../../lua/shared/entityFollow.lua", + "../../lua/shared/createdAtSort.lua", "../../lua/shared/entityFollowLookup.lua" ], "config": { @@ -97,6 +98,8 @@ "../../lua/shared/listValidation.lua", "../../lua/shared/listQuery.lua", "../../lua/shared/entityFollow.lua", + "../../lua/shared/listCursor.lua", + "../../lua/shared/createdAtSort.lua", "../../lua/shared/entityFollowPagination.lua", "../../lua/shared/entityFollowFollowers.lua" ], @@ -122,6 +125,8 @@ "../../lua/shared/collectionProjection.lua", "../../lua/shared/featureProjection.lua", "../../lua/shared/entityFollow.lua", + "../../lua/shared/listCursor.lua", + "../../lua/shared/createdAtSort.lua", "../../lua/shared/entityFollowPagination.lua", "../../lua/shared/entityFollowEntities.lua" ], diff --git a/happyview-modules/modules/feed-reactions/manifest.json b/happyview-modules/modules/feed-reactions/manifest.json index 8eb9a911..c32594bf 100644 --- a/happyview-modules/modules/feed-reactions/manifest.json +++ b/happyview-modules/modules/feed-reactions/manifest.json @@ -70,6 +70,8 @@ "../../lua/shared/listQuery.lua", "../../lua/shared/recordView.lua", "../../lua/shared/actorView.lua", + "../../lua/shared/listCursor.lua", + "../../lua/shared/createdAtSort.lua", "../../lua/shared/feedReaction.lua" ], "config": { "script_type": "lua", "description": "Feed reaction API handler" }, @@ -89,6 +91,8 @@ "../../lua/shared/listQuery.lua", "../../lua/shared/recordView.lua", "../../lua/shared/actorView.lua", + "../../lua/shared/listCursor.lua", + "../../lua/shared/createdAtSort.lua", "../../lua/shared/feedReaction.lua" ], "config": { "script_type": "lua", "description": "Feed reaction API handler" }, @@ -108,6 +112,8 @@ "../../lua/shared/listQuery.lua", "../../lua/shared/recordView.lua", "../../lua/shared/actorView.lua", + "../../lua/shared/listCursor.lua", + "../../lua/shared/createdAtSort.lua", "../../lua/shared/feedReaction.lua" ], "config": { "script_type": "lua", "description": "Feed reaction API handler" }, @@ -127,6 +133,8 @@ "../../lua/shared/listQuery.lua", "../../lua/shared/recordView.lua", "../../lua/shared/actorView.lua", + "../../lua/shared/listCursor.lua", + "../../lua/shared/createdAtSort.lua", "../../lua/shared/feedReaction.lua" ], "config": { "script_type": "lua", "description": "Feed reaction API handler" }, diff --git a/happyview-modules/modules/funding/manifest.json b/happyview-modules/modules/funding/manifest.json index a1dae3f8..8713d36b 100644 --- a/happyview-modules/modules/funding/manifest.json +++ b/happyview-modules/modules/funding/manifest.json @@ -54,6 +54,10 @@ "../../lua/shared/actorView.lua", "../../lua/shared/fundingReceipt.lua", "../../lua/shared/nsidValidation.lua", + "../../lua/shared/listQuery.lua", + "../../lua/shared/listCursor.lua", + "../../lua/shared/listWhere.lua", + "../../lua/shared/createdAtSort.lua", "../../lua/shared/fundingReceiptList.lua" ], "config": { From dbf7899e1bfba507c9ba7421a854af880755176b Mon Sep 17 00:00:00 2001 From: kzoeps Date: Wed, 18 Nov 2026 18:24:02 +0600 Subject: [PATCH 2/2] lua: use query.lua helpers in collection bundles; migrate getEntityFollows sort key --- .../lua/endpoints/getCollection.lua | 47 ++++++++-------- .../lua/endpoints/getEntityFollows.lua | 17 +++++- .../lua/endpoints/listCollections.lua | 55 +++++++------------ .../lua/endpoints/searchCollections.lua | 55 +++++++------------ happyview-modules/lua/shared/collection.lua | 20 ------- .../lua/shared/collectionList.lua | 35 +++++++----- happyview-modules/lua/src/getCollection.lua | 6 +- .../lua/src/getEntityFollows.lua | 2 +- .../modules/collection/manifest.json | 1 + .../modules/entity-follow/manifest.json | 1 + 10 files changed, 106 insertions(+), 133 deletions(-) diff --git a/happyview-modules/lua/endpoints/getCollection.lua b/happyview-modules/lua/endpoints/getCollection.lua index 7e76a4f7..06511fcb 100644 --- a/happyview-modules/lua/endpoints/getCollection.lua +++ b/happyview-modules/lua/endpoints/getCollection.lua @@ -6,6 +6,27 @@ local function valid_did(value) return true end +local function invalid(message) + error("InvalidRequest: " .. message, 0) +end + +local function keys_only(values, allowed, unknown_message_prefix) + for key in pairs(values) do + if not allowed[key] then + invalid(unknown_message_prefix and (unknown_message_prefix .. key) or "unknown query parameter") + end + end +end + +local function scalar(params, key) + local value = params[key] + if value == nil then return nil end + if type(value) ~= "string" and type(value) ~= "number" then + invalid(key .. " must occur once") + end + return tostring(value) +end + local function valid_record_key(value) return #value >= 1 and #value <= 512 and value ~= "." and value ~= ".." and not value:find("[^%w_~%.:%-]") @@ -208,26 +229,6 @@ end local COLLECTION = "org.hypercerts.collection" -local function collection_invalid(message) - error("InvalidRequest: " .. message, 0) -end - -local function collection_keys_only(values, allowed) - for key in pairs(values) do - if not allowed[key] then collection_invalid("unknown query parameter: " .. key) end - end -end - -local function collection_scalar(values, key) - local value = values[key] - if value == nil then return nil end - if key == "hasOrganizationRecord" and type(value) == "boolean" then return tostring(value) end - if type(value) ~= "string" and type(value) ~= "number" then - collection_invalid(key .. " must occur once") - end - return tostring(value) -end - local function collection_valid_record_uri(value) if type(value) ~= "string" or #value > 8192 then return false end return valid_record_uri(value) @@ -253,11 +254,11 @@ local function collection_hydrate(views, omit_invalid) end function handle() - collection_keys_only(params, { uri = true }) - local uri = collection_scalar(params, "uri") + keys_only(params, { uri = true }, "unknown query parameter: ") + local uri = scalar(params, "uri") local valid, collection = collection_valid_record_uri(uri) if not uri or not valid or collection ~= COLLECTION then - collection_invalid("uri must be a full org.hypercerts.collection AT-URI with a DID authority") + invalid("uri must be a full org.hypercerts.collection AT-URI with a DID authority") end local rows = collection_query( diff --git a/happyview-modules/lua/endpoints/getEntityFollows.lua b/happyview-modules/lua/endpoints/getEntityFollows.lua index 61cdc76d..20df09b7 100644 --- a/happyview-modules/lua/endpoints/getEntityFollows.lua +++ b/happyview-modules/lua/endpoints/getEntityFollows.lua @@ -39,6 +39,21 @@ local function valid_record_uri(value) return true, collection, authority end +local CREATED_AT_SORT_PATTERN = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]([.][0-9]+)?(Z|[+-]([01][0-9]|2[0-3]):[0-5][0-9])$" + +-- Sort key for a happyview_records row: the record's own createdAt when it is a +-- valid timestamp, otherwise COALESCE(indexed_at, created_at). `alias` is the +-- table alias; nil selects unqualified columns. +local function created_at_sort(alias) + local prefix = alias and (alias .. ".") or "" + local record, indexed_at, created_at = prefix .. "record", prefix .. "indexed_at", prefix .. "created_at" + local created = record .. "::jsonb->>'createdAt'" + return "CASE WHEN jsonb_typeof(" .. record .. "::jsonb->'createdAt') = 'string' AND " .. created .. + " ~ '" .. CREATED_AT_SORT_PATTERN .. "' AND " .. created .. " !~ '-00:00$' AND pg_input_is_valid(" .. created .. + ", 'timestamptz') THEN (" .. created .. ")::timestamptz ELSE COALESCE(" .. indexed_at .. + "::timestamptz, " .. created_at .. "::timestamptz) END" +end + local ENTITY_FOLLOW = "app.certified.graph.entityFollow" local ENTITY_FOLLOW_NULL = json.decode("null") @@ -212,7 +227,7 @@ local function get_entity_follows() .. "indexed_at::text AS indexed_at, record::text AS record FROM requested " .. "JOIN happyview_records ON collection = $1 AND did = $2 " .. "AND record::jsonb->'subject'->>'uri' = requested.target " - .. "CROSS JOIN LATERAL (SELECT " .. entity_follow_sort_key() .. " AS sort_at) sorted " + .. "CROSS JOIN LATERAL (SELECT " .. created_at_sort() .. " AS sort_at) sorted " .. "ORDER BY requested.target, sorted.sort_at ASC, uri ASC", values) diff --git a/happyview-modules/lua/endpoints/listCollections.lua b/happyview-modules/lua/endpoints/listCollections.lua index a0b04803..b343ad34 100644 --- a/happyview-modules/lua/endpoints/listCollections.lua +++ b/happyview-modules/lua/endpoints/listCollections.lua @@ -208,26 +208,6 @@ end local COLLECTION = "org.hypercerts.collection" -local function collection_invalid(message) - error("InvalidRequest: " .. message, 0) -end - -local function collection_keys_only(values, allowed) - for key in pairs(values) do - if not allowed[key] then collection_invalid("unknown query parameter: " .. key) end - end -end - -local function collection_scalar(values, key) - local value = values[key] - if value == nil then return nil end - if key == "hasOrganizationRecord" and type(value) == "boolean" then return tostring(value) end - if type(value) ~= "string" and type(value) ~= "number" then - collection_invalid(key .. " must occur once") - end - return tostring(value) -end - local function collection_valid_record_uri(value) if type(value) ~= "string" or #value > 8192 then return false end return valid_record_uri(value) @@ -367,6 +347,11 @@ local function add_in(where, binds, column, items) where[#where + 1] = column .. " IN (" .. table.concat(marks, ", ") .. ")" end +local function collection_scalar(values, key) + if key == "hasOrganizationRecord" and type(values[key]) == "boolean" then return tostring(values[key]) end + return scalar(values, key) +end + local function collection_array(key, kind) local value = params[key] if value == nil then return nil end @@ -378,40 +363,40 @@ local function collection_array(key, kind) local count = 0 for index in pairs(value) do if type(index) ~= "number" or index < 1 or index % 1 ~= 0 then - collection_invalid(key .. " must use repeated query values") + invalid(key .. " must use repeated query values") end count = count + 1 end - if count ~= #value then collection_invalid(key .. " must use repeated query values") end + if count ~= #value then invalid(key .. " must use repeated query values") end for index = 1, count do - if type(value[index]) ~= "string" then collection_invalid(key .. " entries must be strings") end + if type(value[index]) ~= "string" then invalid(key .. " entries must be strings") end supplied[#supplied + 1] = value[index] end else - collection_invalid(key .. " must be a string or repeated string parameter") + invalid(key .. " must be a string or repeated string parameter") end - if #supplied > 100 then collection_invalid(key .. " accepts at most 100 values") end + if #supplied > 100 then invalid(key .. " accepts at most 100 values") end local unique, seen = {}, {} for _, item in ipairs(supplied) do if kind == "did" and not valid_did(item) then - collection_invalid("each authors value must be a valid DID; resolve handles to DIDs first") + invalid("each authors value must be a valid DID; resolve handles to DIDs first") elseif kind == "collectionUri" then local valid, collection = collection_valid_record_uri(item) if not valid or collection ~= COLLECTION then - collection_invalid("each uris value must be a full org.hypercerts.collection AT-URI with a DID authority") + invalid("each uris value must be a full org.hypercerts.collection AT-URI with a DID authority") end elseif kind == "recordUri" then if not collection_valid_record_uri(item) then - collection_invalid("each " .. key .. " value must be a full AT-URI with a DID authority") + invalid("each " .. key .. " value must be a full AT-URI with a DID authority") end elseif kind == "tagUri" then local valid, collection = collection_valid_record_uri(item) if not valid or collection ~= "org.hypercerts.vocab.tag" then - collection_invalid("each tagUris value must be a full org.hypercerts.vocab.tag AT-URI with a DID authority") + invalid("each tagUris value must be a full org.hypercerts.vocab.tag AT-URI with a DID authority") end elseif kind == "type" and #item > 64 then - collection_invalid("types entries must be at most 64 UTF-8 bytes") + invalid("types entries must be at most 64 UTF-8 bytes") end if not seen[item] then seen[item] = true @@ -540,7 +525,7 @@ local function collection_list_response(search_enabled) cursor = true, } if search_enabled then allowed.search = true end - collection_keys_only(params, allowed) + keys_only(params, allowed, "unknown query parameter: ") local authors = collection_array("authors", "did") local has_organization_record = collection_scalar(params, "hasOrganizationRecord") @@ -550,7 +535,7 @@ local function collection_list_response(search_enabled) elseif has_organization_record == "false" then has_organization_record = false else - collection_invalid("hasOrganizationRecord must be true or false") + invalid("hasOrganizationRecord must be true or false") end end local types = collection_array("types", "type") @@ -559,14 +544,14 @@ local function collection_list_response(search_enabled) local tag_uris = collection_array("tagUris", "tagUri") local search if search_enabled then - search = collection_scalar(params, "search") - if search == nil then collection_invalid("search is required") end + search = scalar(params, "search") + if search == nil then invalid("search is required") end search = search:match("^%s*(.-)%s*$") if search == "" then search = nil end end local direction = parse_sort_direction(params) local limit = parse_list_limit(params) - local cursor = collection_list_cursor_decode(collection_scalar(params, "cursor"), direction) + local cursor = collection_list_cursor_decode(scalar(params, "cursor"), direction) local collections, next_cursor = collection_list_query({ authors = authors, hasOrganizationRecord = has_organization_record, diff --git a/happyview-modules/lua/endpoints/searchCollections.lua b/happyview-modules/lua/endpoints/searchCollections.lua index 16b79369..a9a6dafc 100644 --- a/happyview-modules/lua/endpoints/searchCollections.lua +++ b/happyview-modules/lua/endpoints/searchCollections.lua @@ -208,26 +208,6 @@ end local COLLECTION = "org.hypercerts.collection" -local function collection_invalid(message) - error("InvalidRequest: " .. message, 0) -end - -local function collection_keys_only(values, allowed) - for key in pairs(values) do - if not allowed[key] then collection_invalid("unknown query parameter: " .. key) end - end -end - -local function collection_scalar(values, key) - local value = values[key] - if value == nil then return nil end - if key == "hasOrganizationRecord" and type(value) == "boolean" then return tostring(value) end - if type(value) ~= "string" and type(value) ~= "number" then - collection_invalid(key .. " must occur once") - end - return tostring(value) -end - local function collection_valid_record_uri(value) if type(value) ~= "string" or #value > 8192 then return false end return valid_record_uri(value) @@ -367,6 +347,11 @@ local function add_in(where, binds, column, items) where[#where + 1] = column .. " IN (" .. table.concat(marks, ", ") .. ")" end +local function collection_scalar(values, key) + if key == "hasOrganizationRecord" and type(values[key]) == "boolean" then return tostring(values[key]) end + return scalar(values, key) +end + local function collection_array(key, kind) local value = params[key] if value == nil then return nil end @@ -378,40 +363,40 @@ local function collection_array(key, kind) local count = 0 for index in pairs(value) do if type(index) ~= "number" or index < 1 or index % 1 ~= 0 then - collection_invalid(key .. " must use repeated query values") + invalid(key .. " must use repeated query values") end count = count + 1 end - if count ~= #value then collection_invalid(key .. " must use repeated query values") end + if count ~= #value then invalid(key .. " must use repeated query values") end for index = 1, count do - if type(value[index]) ~= "string" then collection_invalid(key .. " entries must be strings") end + if type(value[index]) ~= "string" then invalid(key .. " entries must be strings") end supplied[#supplied + 1] = value[index] end else - collection_invalid(key .. " must be a string or repeated string parameter") + invalid(key .. " must be a string or repeated string parameter") end - if #supplied > 100 then collection_invalid(key .. " accepts at most 100 values") end + if #supplied > 100 then invalid(key .. " accepts at most 100 values") end local unique, seen = {}, {} for _, item in ipairs(supplied) do if kind == "did" and not valid_did(item) then - collection_invalid("each authors value must be a valid DID; resolve handles to DIDs first") + invalid("each authors value must be a valid DID; resolve handles to DIDs first") elseif kind == "collectionUri" then local valid, collection = collection_valid_record_uri(item) if not valid or collection ~= COLLECTION then - collection_invalid("each uris value must be a full org.hypercerts.collection AT-URI with a DID authority") + invalid("each uris value must be a full org.hypercerts.collection AT-URI with a DID authority") end elseif kind == "recordUri" then if not collection_valid_record_uri(item) then - collection_invalid("each " .. key .. " value must be a full AT-URI with a DID authority") + invalid("each " .. key .. " value must be a full AT-URI with a DID authority") end elseif kind == "tagUri" then local valid, collection = collection_valid_record_uri(item) if not valid or collection ~= "org.hypercerts.vocab.tag" then - collection_invalid("each tagUris value must be a full org.hypercerts.vocab.tag AT-URI with a DID authority") + invalid("each tagUris value must be a full org.hypercerts.vocab.tag AT-URI with a DID authority") end elseif kind == "type" and #item > 64 then - collection_invalid("types entries must be at most 64 UTF-8 bytes") + invalid("types entries must be at most 64 UTF-8 bytes") end if not seen[item] then seen[item] = true @@ -540,7 +525,7 @@ local function collection_list_response(search_enabled) cursor = true, } if search_enabled then allowed.search = true end - collection_keys_only(params, allowed) + keys_only(params, allowed, "unknown query parameter: ") local authors = collection_array("authors", "did") local has_organization_record = collection_scalar(params, "hasOrganizationRecord") @@ -550,7 +535,7 @@ local function collection_list_response(search_enabled) elseif has_organization_record == "false" then has_organization_record = false else - collection_invalid("hasOrganizationRecord must be true or false") + invalid("hasOrganizationRecord must be true or false") end end local types = collection_array("types", "type") @@ -559,14 +544,14 @@ local function collection_list_response(search_enabled) local tag_uris = collection_array("tagUris", "tagUri") local search if search_enabled then - search = collection_scalar(params, "search") - if search == nil then collection_invalid("search is required") end + search = scalar(params, "search") + if search == nil then invalid("search is required") end search = search:match("^%s*(.-)%s*$") if search == "" then search = nil end end local direction = parse_sort_direction(params) local limit = parse_list_limit(params) - local cursor = collection_list_cursor_decode(collection_scalar(params, "cursor"), direction) + local cursor = collection_list_cursor_decode(scalar(params, "cursor"), direction) local collections, next_cursor = collection_list_query({ authors = authors, hasOrganizationRecord = has_organization_record, diff --git a/happyview-modules/lua/shared/collection.lua b/happyview-modules/lua/shared/collection.lua index 820ca04d..03280698 100644 --- a/happyview-modules/lua/shared/collection.lua +++ b/happyview-modules/lua/shared/collection.lua @@ -1,25 +1,5 @@ local COLLECTION = "org.hypercerts.collection" -local function collection_invalid(message) - error("InvalidRequest: " .. message, 0) -end - -local function collection_keys_only(values, allowed) - for key in pairs(values) do - if not allowed[key] then collection_invalid("unknown query parameter: " .. key) end - end -end - -local function collection_scalar(values, key) - local value = values[key] - if value == nil then return nil end - if key == "hasOrganizationRecord" and type(value) == "boolean" then return tostring(value) end - if type(value) ~= "string" and type(value) ~= "number" then - collection_invalid(key .. " must occur once") - end - return tostring(value) -end - local function collection_valid_record_uri(value) if type(value) ~= "string" or #value > 8192 then return false end return valid_record_uri(value) diff --git a/happyview-modules/lua/shared/collectionList.lua b/happyview-modules/lua/shared/collectionList.lua index 8520fc91..0a4f0524 100644 --- a/happyview-modules/lua/shared/collectionList.lua +++ b/happyview-modules/lua/shared/collectionList.lua @@ -1,3 +1,8 @@ +local function collection_scalar(values, key) + if key == "hasOrganizationRecord" and type(values[key]) == "boolean" then return tostring(values[key]) end + return scalar(values, key) +end + local function collection_array(key, kind) local value = params[key] if value == nil then return nil end @@ -9,40 +14,40 @@ local function collection_array(key, kind) local count = 0 for index in pairs(value) do if type(index) ~= "number" or index < 1 or index % 1 ~= 0 then - collection_invalid(key .. " must use repeated query values") + invalid(key .. " must use repeated query values") end count = count + 1 end - if count ~= #value then collection_invalid(key .. " must use repeated query values") end + if count ~= #value then invalid(key .. " must use repeated query values") end for index = 1, count do - if type(value[index]) ~= "string" then collection_invalid(key .. " entries must be strings") end + if type(value[index]) ~= "string" then invalid(key .. " entries must be strings") end supplied[#supplied + 1] = value[index] end else - collection_invalid(key .. " must be a string or repeated string parameter") + invalid(key .. " must be a string or repeated string parameter") end - if #supplied > 100 then collection_invalid(key .. " accepts at most 100 values") end + if #supplied > 100 then invalid(key .. " accepts at most 100 values") end local unique, seen = {}, {} for _, item in ipairs(supplied) do if kind == "did" and not valid_did(item) then - collection_invalid("each authors value must be a valid DID; resolve handles to DIDs first") + invalid("each authors value must be a valid DID; resolve handles to DIDs first") elseif kind == "collectionUri" then local valid, collection = collection_valid_record_uri(item) if not valid or collection ~= COLLECTION then - collection_invalid("each uris value must be a full org.hypercerts.collection AT-URI with a DID authority") + invalid("each uris value must be a full org.hypercerts.collection AT-URI with a DID authority") end elseif kind == "recordUri" then if not collection_valid_record_uri(item) then - collection_invalid("each " .. key .. " value must be a full AT-URI with a DID authority") + invalid("each " .. key .. " value must be a full AT-URI with a DID authority") end elseif kind == "tagUri" then local valid, collection = collection_valid_record_uri(item) if not valid or collection ~= "org.hypercerts.vocab.tag" then - collection_invalid("each tagUris value must be a full org.hypercerts.vocab.tag AT-URI with a DID authority") + invalid("each tagUris value must be a full org.hypercerts.vocab.tag AT-URI with a DID authority") end elseif kind == "type" and #item > 64 then - collection_invalid("types entries must be at most 64 UTF-8 bytes") + invalid("types entries must be at most 64 UTF-8 bytes") end if not seen[item] then seen[item] = true @@ -171,7 +176,7 @@ local function collection_list_response(search_enabled) cursor = true, } if search_enabled then allowed.search = true end - collection_keys_only(params, allowed) + keys_only(params, allowed, "unknown query parameter: ") local authors = collection_array("authors", "did") local has_organization_record = collection_scalar(params, "hasOrganizationRecord") @@ -181,7 +186,7 @@ local function collection_list_response(search_enabled) elseif has_organization_record == "false" then has_organization_record = false else - collection_invalid("hasOrganizationRecord must be true or false") + invalid("hasOrganizationRecord must be true or false") end end local types = collection_array("types", "type") @@ -190,14 +195,14 @@ local function collection_list_response(search_enabled) local tag_uris = collection_array("tagUris", "tagUri") local search if search_enabled then - search = collection_scalar(params, "search") - if search == nil then collection_invalid("search is required") end + search = scalar(params, "search") + if search == nil then invalid("search is required") end search = search:match("^%s*(.-)%s*$") if search == "" then search = nil end end local direction = parse_sort_direction(params) local limit = parse_list_limit(params) - local cursor = collection_list_cursor_decode(collection_scalar(params, "cursor"), direction) + local cursor = collection_list_cursor_decode(scalar(params, "cursor"), direction) local collections, next_cursor = collection_list_query({ authors = authors, hasOrganizationRecord = has_organization_record, diff --git a/happyview-modules/lua/src/getCollection.lua b/happyview-modules/lua/src/getCollection.lua index 45233f63..98d5dd48 100644 --- a/happyview-modules/lua/src/getCollection.lua +++ b/happyview-modules/lua/src/getCollection.lua @@ -1,9 +1,9 @@ function handle() - collection_keys_only(params, { uri = true }) - local uri = collection_scalar(params, "uri") + keys_only(params, { uri = true }, "unknown query parameter: ") + local uri = scalar(params, "uri") local valid, collection = collection_valid_record_uri(uri) if not uri or not valid or collection ~= COLLECTION then - collection_invalid("uri must be a full org.hypercerts.collection AT-URI with a DID authority") + invalid("uri must be a full org.hypercerts.collection AT-URI with a DID authority") end local rows = collection_query( diff --git a/happyview-modules/lua/src/getEntityFollows.lua b/happyview-modules/lua/src/getEntityFollows.lua index fc92e944..ef021259 100644 --- a/happyview-modules/lua/src/getEntityFollows.lua +++ b/happyview-modules/lua/src/getEntityFollows.lua @@ -18,7 +18,7 @@ local function get_entity_follows() .. "indexed_at::text AS indexed_at, record::text AS record FROM requested " .. "JOIN happyview_records ON collection = $1 AND did = $2 " .. "AND record::jsonb->'subject'->>'uri' = requested.target " - .. "CROSS JOIN LATERAL (SELECT " .. entity_follow_sort_key() .. " AS sort_at) sorted " + .. "CROSS JOIN LATERAL (SELECT " .. created_at_sort() .. " AS sort_at) sorted " .. "ORDER BY requested.target, sorted.sort_at ASC, uri ASC", values) diff --git a/happyview-modules/modules/collection/manifest.json b/happyview-modules/modules/collection/manifest.json index caa471b3..2bfde0d6 100644 --- a/happyview-modules/modules/collection/manifest.json +++ b/happyview-modules/modules/collection/manifest.json @@ -7,6 +7,7 @@ "sourcePath": "../../lua/src/getCollection.lua", "sharedSourcePaths": [ "../../lua/shared/didValidation.lua", + "../../lua/shared/query.lua", "../../lua/shared/recordIdentifier.lua", "../../lua/shared/collectionProjection.lua", "../../lua/shared/collection.lua" diff --git a/happyview-modules/modules/entity-follow/manifest.json b/happyview-modules/modules/entity-follow/manifest.json index 69ad1b3f..d5c40dd0 100644 --- a/happyview-modules/modules/entity-follow/manifest.json +++ b/happyview-modules/modules/entity-follow/manifest.json @@ -73,6 +73,7 @@ "../../lua/shared/didValidation.lua", "../../lua/shared/query.lua", "../../lua/shared/recordIdentifier.lua", + "../../lua/shared/createdAtSort.lua", "../../lua/shared/entityFollow.lua", "../../lua/shared/batchRequestTarget.lua", "../../lua/shared/followBatch.lua",