link/html: ignore description if similar/too similar to titlemaster

1 files changed, 9 insertions, 4 deletions

diff --git a/lib/plugins/link/html.ex b/lib/plugins/link/html.ex
index 1173526..7df1bfc 100644
--- a/lib/plugins/link/html.ex
+++ b/lib/plugins/link/html.ex
@@ -30,9 +30,10 @@ defmodule Nola.Plugins.Link.HTML do
   defp generate_text_from_opengraph(url, html, opengraph) do
     itemprops = collect_itemprops(html)
     prefix = collect_prefix_and_site_name(url, opengraph, itemprops)
-    description = collect_description(opengraph, itemprops, 400)
+    title = Map.get(opengraph, "title")
+    description = collect_description(opengraph, itemprops, title, 400)
 
-    [clean_text("#{prefix}#{Map.get(opengraph, "title")}")] ++ description
+    [clean_text("#{prefix}#{title}")] ++ description
   end
 
   defp collect_title(html) do
@@ -114,11 +115,15 @@ defmodule Nola.Plugins.Link.HTML do
     if section, do: ": #{section}", else: ""
   end
 
-  defp collect_description(opengraph, itemprops, max_length) do
+  defp collect_description(opengraph, itemprops, title, max_length) do
     date = get_formatted_date(opengraph, itemprops)
     description = transform_description(Map.get(opengraph, "description"), max_length)
 
-    Nola.Irc.Message.splitlong(clean_text("#{date}#{description}"))
+    cond do
+      title == description -> date
+      String.jaro_distance(title, description) > 0.8 -> date
+      true -> Nola.Irc.Message.splitlong(clean_text("#{date}#{description}"))
+    end
   end
 
   defp get_formatted_date(opengraph, itemprops) do