From 1b0e59bb139f83d1f027ece08606446c30d35ae5 Mon Sep 17 00:00:00 2001 From: Holger Weiss Date: Sat, 21 Aug 2021 12:29:37 +0200 Subject: PubSub: Support unlimited number of items Allow for setting the mod_pubsub option 'max_items_node' to 'unlimited'. If clients then request a 'max_items' limit of 'max', old items aren't deleted when publishing new ones. Thanks to Ammonit Measurement GmbH for sponsoring this work. --- src/node_flat_sql.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/node_flat_sql.erl') diff --git a/src/node_flat_sql.erl b/src/node_flat_sql.erl index 1e197a51d..1309c2886 100644 --- a/src/node_flat_sql.erl +++ b/src/node_flat_sql.erl @@ -247,7 +247,8 @@ publish_item(Nidx, Publisher, PublishModel, MaxItems, ItemId, Payload, or (Subscribed == true)) -> {error, xmpp:err_forbidden()}; true -> - if MaxItems > 0 -> + if MaxItems > 0; + MaxItems == unlimited -> Now = erlang:timestamp(), case get_item(Nidx, ItemId) of {result, #pubsub_item{creation = {_, GenKey}} = OldItem} -> -- cgit v1.2.3 From 29751a6174d181f80bbcde7c92f8cc958eee85a1 Mon Sep 17 00:00:00 2001 From: Holger Weiss Date: Sat, 21 Aug 2021 20:02:58 +0200 Subject: PubSub: Optimize publishing on large nodes (SQL) Avoid an unnecessary SQL query while publishing an item on a PubSub node without 'max_items' limit. The query in question can be expensive if the node has a large number of items. Thanks to Ammonit Measurement GmbH for sponsoring this work. --- src/node_flat_sql.erl | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'src/node_flat_sql.erl') diff --git a/src/node_flat_sql.erl b/src/node_flat_sql.erl index 1309c2886..5033eda51 100644 --- a/src/node_flat_sql.erl +++ b/src/node_flat_sql.erl @@ -259,14 +259,14 @@ publish_item(Nidx, Publisher, PublishModel, MaxItems, ItemId, Payload, {result, _} -> {error, xmpp:err_forbidden()}; _ -> - Items = [ItemId | itemids(Nidx, GenKey)], - {result, {_NI, OI}} = remove_extra_items(Nidx, MaxItems, Items), + OldIds = maybe_remove_extra_items(Nidx, MaxItems, + GenKey, ItemId), set_item(#pubsub_item{ itemid = {ItemId, Nidx}, creation = {Now, GenKey}, modification = {Now, SubKey}, payload = Payload}), - {result, {default, broadcast, OI}} + {result, {default, broadcast, OldIds}} end; true -> {result, {default, broadcast, []}} @@ -934,6 +934,16 @@ update_subscription(Nidx, JID, Subscription) -> "-affiliation='n'" ]). +-spec maybe_remove_extra_items(mod_pubsub:nodeIdx(), + non_neg_integer() | unlimited, ljid(), + mod_pubsub:itemId()) -> [mod_pubsub:itemId()]. +maybe_remove_extra_items(_Nidx, unlimited, _GenKey, _ItemId) -> + []; +maybe_remove_extra_items(Nidx, MaxItems, GenKey, ItemId) -> + ItemIds = [ItemId | itemids(Nidx, GenKey)], + {result, {_NewIds, OldIds}} = remove_extra_items(Nidx, MaxItems, ItemIds), + OldIds. + -spec decode_jid(SJID :: binary()) -> ljid(). decode_jid(SJID) -> jid:tolower(jid:decode(SJID)). -- cgit v1.2.3 From 8d5025076f53abc0474c86a0f879ff1736714844 Mon Sep 17 00:00:00 2001 From: Holger Weiss Date: Sun, 22 Aug 2021 12:44:50 +0200 Subject: PubSub: Add delete_old_pubsub_items command Add a command for keeping only the specified number of items on each node and removing all older items. This might be especially useful if nodes may be configured to have no 'max_items' limit. Thanks to Ammonit Measurement GmbH for sponsoring this work. --- src/node_flat_sql.erl | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) (limited to 'src/node_flat_sql.erl') diff --git a/src/node_flat_sql.erl b/src/node_flat_sql.erl index 5033eda51..724958eb1 100644 --- a/src/node_flat_sql.erl +++ b/src/node_flat_sql.erl @@ -40,9 +40,10 @@ -include("translate.hrl"). -export([init/3, terminate/2, options/0, features/0, - create_node_permission/6, create_node/2, delete_node/1, - purge_node/2, subscribe_node/8, unsubscribe_node/4, - publish_item/7, delete_item/4, remove_extra_items/3, + create_node_permission/6, create_node/2, delete_node/1, purge_node/2, + subscribe_node/8, unsubscribe_node/4, + publish_item/7, delete_item/4, + remove_extra_items/2, remove_extra_items/3, get_entity_affiliations/2, get_node_affiliations/1, get_affiliation/2, set_affiliation/3, get_entity_subscriptions/2, get_node_subscriptions/1, @@ -273,6 +274,9 @@ publish_item(Nidx, Publisher, PublishModel, MaxItems, ItemId, Payload, end end. +remove_extra_items(Nidx, MaxItems) -> + remove_extra_items(Nidx, MaxItems, itemids(Nidx)). + remove_extra_items(_Nidx, unlimited, ItemIds) -> {result, {ItemIds, []}}; remove_extra_items(Nidx, MaxItems, ItemIds) -> @@ -863,6 +867,18 @@ first_in_list(Pred, [H | T]) -> _ -> first_in_list(Pred, T) end. +itemids(Nidx) -> + case catch + ejabberd_sql:sql_query_t( + ?SQL("select @(itemid)s from pubsub_item where " + "nodeid=%(Nidx)d order by modification desc")) + of + {selected, RItems} -> + [ItemId || {ItemId} <- RItems]; + _ -> + [] + end. + itemids(Nidx, {_U, _S, _R} = JID) -> SJID = encode_jid(JID), SJIDLike = <<(encode_jid_like(JID))/binary, "/%">>, -- cgit v1.2.3 From c952cc420b345a0e43fc4e6d819b3edb3ac12cb1 Mon Sep 17 00:00:00 2001 From: Holger Weiss Date: Mon, 23 Aug 2021 21:28:15 +0200 Subject: node_flat_sql: Avoid catch-all clauses for RSM Explicitly catch invalid and timestamps specified by clients in RSM queries, but crash on other errors, rather than silently ignoring those. --- src/node_flat_sql.erl | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'src/node_flat_sql.erl') diff --git a/src/node_flat_sql.erl b/src/node_flat_sql.erl index 724958eb1..240dc3760 100644 --- a/src/node_flat_sql.erl +++ b/src/node_flat_sql.erl @@ -1064,15 +1064,14 @@ rsm_page(Count, Index, Offset, Items) -> last = Last}. encode_stamp(Stamp) -> - case catch xmpp_util:decode_timestamp(Stamp) of - {MS,S,US} -> encode_now({MS,S,US}); - _ -> Stamp + try xmpp_util:decode_timestamp(Stamp) of + Now -> + encode_now(Now) + catch _:{bad_timestamp, _} -> + Stamp % We should return a proper error to the client instead. end. decode_stamp(Stamp) -> - case catch xmpp_util:encode_timestamp(decode_now(Stamp)) of - TimeStamp when is_binary(TimeStamp) -> TimeStamp; - _ -> Stamp - end. + xmpp_util:encode_timestamp(decode_now(Stamp)). encode_now({T1, T2, T3}) -> <<(misc:i2l(T1, 6))/binary, ":", -- cgit v1.2.3