link: post_* callbacks; html & pdftitle.

7 files changed, 166 insertions, 0 deletions

diff --git a/lib/lsg_irc/link_plugin/github.ex b/lib/lsg_irc/link_plugin/github.ex
index c7444c2..19be89b 100644
--- a/lib/lsg_irc/link_plugin/github.ex
+++ b/lib/lsg_irc/link_plugin/github.ex
@@ -1,6 +1,7 @@
 defmodule LSG.IRC.LinkPlugin.Github do
   @behaviour LSG.IRC.LinkPlugin
 
+  @impl true
   def match(uri = %URI{host: "github.com", path: path}, _) do
     case String.split(path, "/") do
       ["", user, repo] ->
@@ -12,6 +13,10 @@ defmodule LSG.IRC.LinkPlugin.Github do
 
   def match(_, _), do: false
 
+  @impl true
+  def post_match(_, _, _, _), do: false
+
+  @impl true
   def expand(_uri, %{user: user, repo: repo}, _opts) do
     case HTTPoison.get("https://api.github.com/repos/#{user}/#{repo}") do
       {:ok, %HTTPoison.Response{status_code: 200, body: body}} ->
diff --git a/lib/lsg_irc/link_plugin/html.ex b/lib/lsg_irc/link_plugin/html.ex
new file mode 100644
index 0000000..e0e4229
--- /dev/null
+++ b/lib/lsg_irc/link_plugin/html.ex
@@ -0,0 +1,106 @@
+defmodule LSG.IRC.LinkPlugin.HTML do
+  @behaviour LSG.IRC.LinkPlugin
+
+  @impl true
+  def match(_, _), do: false
+
+  @impl true
+  def post_match(_url, "text/html"<>_, _header, _opts) do
+    {:body, nil}
+  end
+  def post_match(_, _, _, _), do: false
+
+  @impl true
+  def post_expand(url, body, _params, _opts) do
+    html = Floki.parse(body)
+    title = collect_title(html)
+    opengraph = collect_open_graph(html)
+    itemprops = collect_itemprops(html)
+    text = if Map.has_key?(opengraph, "title") && Map.has_key?(opengraph, "description") do
+      sitename = if sn = Map.get(opengraph, "site_name") do
+        "#{sn}"
+      else
+        ""
+      end
+      paywall? = if Map.get(opengraph, "article:content_tier", Map.get(itemprops, "article:content_tier", "free")) == "free" do
+        ""
+      else
+        "[paywall] "
+      end
+      section = if section = Map.get(opengraph, "article:section", Map.get(itemprops, "article:section", nil)) do
+        ": #{section}"
+      else
+        ""
+      end
+      date = case DateTime.from_iso8601(Map.get(opengraph, "article:published_time", Map.get(itemprops, "article:published_time", ""))) do
+        {:ok, date, _} ->
+          "#{Timex.format!(date, "%d/%m/%y", :strftime)}. "
+        _ ->
+          ""
+      end
+      uri = URI.parse(url)
+
+      prefix = "#{paywall?}#{Map.get(opengraph, "site_name", uri.host)}#{section}"
+      prefix = unless prefix == "" do
+        "#{prefix} — "
+      else
+          ""
+      end
+      [clean_text("#{prefix}#{Map.get(opengraph, "title")}")] ++ IRC.splitlong(clean_text("#{date}#{Map.get(opengraph, "description")}"))
+    else
+      clean_text(title)
+    end
+    {:ok, text}
+  end
+
+  defp collect_title(html) do
+    case Floki.find(html, "title") do
+    [{"title", [], [title]} | _] ->
+      String.trim(title)
+    _ ->
+      nil
+    end
+  end
+
+  defp collect_open_graph(html) do
+    Enum.reduce(Floki.find(html, "head meta"), %{}, fn(tag, acc) ->
+      case tag do
+        {"meta", values, []} ->
+          name = List.keyfind(values, "property", 0, {nil, nil}) |> elem(1)
+          content = List.keyfind(values, "content", 0, {nil, nil}) |> elem(1)
+          case name do
+           "og:" <> key ->
+              Map.put(acc, key, content)
+            "article:"<>_ ->
+              Map.put(acc, name, content)
+            _other -> acc
+          end
+        _other -> acc
+      end
+    end)
+  end
+
+  defp collect_itemprops(html) do
+    Enum.reduce(Floki.find(html, "[itemprop]"), %{}, fn(tag, acc) ->
+      case tag do
+        {"meta", values, []} ->
+          name = List.keyfind(values, "itemprop", 0, {nil, nil}) |> elem(1)
+          content = List.keyfind(values, "content", 0, {nil, nil}) |> elem(1)
+          case name do
+           "article:" <> key ->
+              Map.put(acc, name, content)
+            _other -> acc
+          end
+        _other -> acc
+      end
+    end)
+  end
+
+  defp clean_text(text) do
+    text
+    |> String.replace("\n", " ")
+    |> HtmlEntities.decode()
+  end
+
+
+end
diff --git a/lib/lsg_irc/link_plugin/imgur.ex b/lib/lsg_irc/link_plugin/imgur.ex
index 1b8173f..41b7e08 100644
--- a/lib/lsg_irc/link_plugin/imgur.ex
+++ b/lib/lsg_irc/link_plugin/imgur.ex
@@ -15,6 +15,7 @@ defmodule LSG.IRC.LinkPlugin.Imgur do
   ```
   """
 
+  @impl true
   def match(uri = %URI{host: "imgur.com", path: "/a/"<>album_id}, _) do
     {true, %{album_id: album_id}}
   end
@@ -27,6 +28,9 @@ defmodule LSG.IRC.LinkPlugin.Imgur do
   end
   def match(_, _), do: false
 
+  @impl true
+  def post_match(_, _, _, _), do: false
+
   def expand(_uri, %{album_id: album_id}, opts) do
     expand_imgur_album(album_id, opts)
   end
diff --git a/lib/lsg_irc/link_plugin/pdf.ex b/lib/lsg_irc/link_plugin/pdf.ex
new file mode 100644
index 0000000..8c4869c
--- /dev/null
+++ b/lib/lsg_irc/link_plugin/pdf.ex
@@ -0,0 +1,39 @@
+defmodule LSG.IRC.LinkPlugin.PDF do
+  require Logger
+  @behaviour LSG.IRC.LinkPlugin
+
+  @impl true
+  def match(_, _), do: false
+
+  @impl true
+  def post_match(_url, "application/pdf"<>_, _header, _opts) do
+    {:file, nil}
+  end
+
+  def post_match(_, _, _, _), do: false
+
+  @impl true
+  def post_expand(url, file, _, _) do
+    case System.cmd("pdftitle", ["-p", file]) do
+      {text, 0} ->
+        text = text
+               |> String.trim()
+
+        if text == "" do
+          :error
+        else
+          basename = Path.basename(url, ".pdf")
+          text = "[#{basename}] " <> text
+               |> String.split("\n")
+          {:ok, text}
+        end
+      {_, 127} ->
+        Logger.error("dependency `pdftitle` is missing, please install it: `pip3 install pdftitle`.")
+        :error
+      {error, code} ->
+        Logger.warn("command `pdftitle` exited with status code #{code}:\n#{inspect error}")
+        :error
+    end
+  end
+
+end
diff --git a/lib/lsg_irc/link_plugin/reddit_plugin.ex b/lib/lsg_irc/link_plugin/reddit.ex
index a7f5235..6fc1723 100644
--- a/lib/lsg_irc/link_plugin/reddit_plugin.ex
+++ b/lib/lsg_irc/link_plugin/reddit.ex
@@ -1,6 +1,7 @@
 defmodule LSG.IRC.LinkPlugin.Reddit do
   @behaviour LSG.IRC.LinkPlugin
 
+  @impl true
   def match(uri = %URI{host: "reddit.com", path: path}, _) do
     case String.split(path, "/") do
       ["", "r", sub, "comments", post_id, _slug] ->
@@ -26,6 +27,10 @@ defmodule LSG.IRC.LinkPlugin.Reddit do
     end
   end
 
+  @impl true
+  def post_match(_, _, _, _), do: false
+
+  @impl true
   def expand(_, %{mode: :sub, sub: sub}, _opts) do
     url = "https://api.reddit.com/r/#{sub}/about"
     case HTTPoison.get(url) do
diff --git a/lib/lsg_irc/link_plugin/twitter.ex b/lib/lsg_irc/link_plugin/twitter.ex
index a6b6e29..e462384 100644
--- a/lib/lsg_irc/link_plugin/twitter.ex
+++ b/lib/lsg_irc/link_plugin/twitter.ex
@@ -33,6 +33,9 @@ defmodule LSG.IRC.LinkPlugin.Twitter do
 
   def match(_, _), do: false
 
+  @impl true
+  def post_match(_, _, _, _), do: false
+
   def expand(_uri, %{status_id: status_id}, opts) do
     expand_tweet(ExTwitter.show(status_id, tweet_mode: "extended"), opts)
   end
diff --git a/lib/lsg_irc/link_plugin/youtube.ex b/lib/lsg_irc/link_plugin/youtube.ex
index ea4f213..b68a86f 100644
--- a/lib/lsg_irc/link_plugin/youtube.ex
+++ b/lib/lsg_irc/link_plugin/youtube.ex
@@ -16,6 +16,7 @@ defmodule LSG.IRC.LinkPlugin.YouTube do
   * `invidious`: Add a link to invidio.us. Default: true.
   """
 
+  @impl true
   def match(uri = %URI{host: yt, path: "/watch", query: "v="<>video_id}, _opts) when yt in ["youtube.com", "www.youtube.com"] do
     {true, %{video_id: video_id}}
   end
@@ -26,7 +27,10 @@ defmodule LSG.IRC.LinkPlugin.YouTube do
 
   def match(_, _), do: false
 
+  @impl true
+  def post_match(_, _, _, _), do: false
 
+  @impl true
   def expand(uri, %{video_id: video_id}, opts) do
     key = Application.get_env(:lsg, :youtube)[:api_key]
     params = %{