HTTPoison (Hackney) Follow Redirect without Reusing Authorization Headers

You can manually do the redirects, adapting the headers/options on each request, but it’s kinda busywork that follow_redirect: true hopes to avoid unless you happen upon a case like this.

Here’s some code I came up with once upon a time to deal with a similar scenario where specific insight was needed to know info about the hops involved in the redirect.

A couple things to point out:

1. You need to build in a safety valve, i.e. a maximum number of redirects, otherwise you could end up in an endless redirect.
2. The redirect targets need to be qualified, e.g. by prepending the full hostname, so this recursive process keeps a little bit if state.

Here’s my code – put together into a single module. Hopefully this is useful to you: you can see where you could manipulate the options between successive requests.

defmodule RecursiveClient do
  @moduledoc """
  This module augments the functionality available in another HTTP client (e.g.
  `HTTPoison`) so we can know more about requests that _redirect_. Specifically,
  we want to know the final URL where a series of redirects landed.

  Whereas `HTTPoison` only discloses the originally requested URL (when `:follow_redirect`
  is true), this module keeps track of *both* the originally requested URL and the
  the ultimate URL that finally yielded a successful response.

  Note: only `:get` requests are currently supported.
  """

  use HTTPoison.Base

  import HTTPClients

  alias HTTPoison.Response

  @default_client HTTPoison
  @hard_limit_max_redirects 10
  @custom_response_header "X-Original-Request-URL"

  @doc """
  See `RecursiveClient.get/3`
  """
  @impl true
  def get(url), do: get(url, [], [])

  @doc """
  See `RecursiveClient.get/3`
  """
  @impl true
  def get(url, headers), do: get(url, headers, [])

  @doc """
  Recursively makes a GET request to the given `url` following any redirects up to
  `max_redirect` times while keeping track of the URL that initiated the sequence
  of requests as well as the URL that eventually yielded content.

  ## Options

    - `:client` an HTTP client module implementing the `HTTPoison.Base`
      callbacks. Default: `#{@default_client}`
    - `:max_redirect` non-negative integer providing a hard-stop on the number
      of redirects this will follow. Default `#{@hard_limit_max_redirects}`

  ## Examples

      iex> RecursiveClient.get("https://blog.ahrefs.com", [], client: HTTPoison, max_redirect: 3)
      {:ok, %HTTPoison.Response{
        body: _,
        headers: [{"#{@custom_response_header}", "https://blog.ahrefs.com"} | _],
        request: _,
        request_url: "https://ahrefs.com/blog/",
        status_code: 200
      }}

      iex> {:ok, response} = RecursiveClient.get("https://blog.ahrefs.com")
      iex> RecursiveClient.requested_resolved(response)
      {:ok, "https://blog.ahrefs.com", "https://ahrefs.com/blog/"}
  """
  @impl true
  def get(url, headers, options) do
    {http_client, options} = Keyword.pop(options, :client, @default_client)
    {max_redirect, options} = Keyword.pop(options, :max_redirect, @hard_limit_max_redirects)
    options = Keyword.put(options, :follow_redirect, false)

    case make_recursive_request({http_client, :get, [url, headers, options]}, max_redirect, 1) do
      {:ok, response} -> {:ok, append_custom_response_header(url, response)}
      {:error, _error} -> {:error, %HTTPoison.Error{id: url, reason: :ehostunreach}}
    end
  end

  defp make_recursive_request(_, max_redirect, n_redirect)
       when n_redirect > max_redirect do
    {:error, "Too many redirects"}
  end

  # The first of the args is assumed to be the URL to request.
  # We shift it off because we may need to replace it with a new URL
  defp make_recursive_request(
         {client, function, [url | other_args] = args},
         max_redirect,
         n_redirect
       ) do
    case is_valid_url?(url) do
      true ->
        client
        |> make_request(function, args)
        |> handle_response(fn new_request_url ->
          make_recursive_request(
            {client, function, [new_request_url | other_args]},
            max_redirect,
            n_redirect + 1
          )
        end)

      false ->
        {:error, "Invalid url: #{inspect(url)}"}
    end
  end

  defp handle_response(
         {:error, %Response{status_code: status_code} = response},
         redirect_callback
       )
       when status_code >= 300 and status_code <= 399 do
    case target_url(response) do
      {:ok, new_request_url} ->
        redirect_callback.(new_request_url)

      # Couldn't determine the new request URL
      {:error, error} ->
        {:error, error}
    end
  end

  defp handle_response(response, _), do: response

  # Adds a custom response header to the Response: we stash the info
  # there so it is in a place that is compatible with the defined behaviour.
  defp append_custom_response_header(original_url, %Response{headers: headers} = response) do
    Map.put(response, :headers, [{@custom_response_header, original_url}] ++ headers)
  end

  @doc """
  Takes in a URL and determines if URL is valid or invalid.
  See https://github.com/johno/is_url/blob/master/lib/is_url.ex

  ## Examples
      iex> RecursiveClient.is_valid_url?("https://blog.ahrefs.com")
      true

      iex> RecursiveClient.is_valid_url?("invalid_url")
      false
  """
  @spec is_valid_url?(binary) :: boolean()
  def is_valid_url?(url) when is_binary(url) do
    case URI.parse(url) do
      %URI{scheme: scheme} when scheme not in ["https", "http"] -> false
      %URI{host: host} when host in ["", nil] -> false
      _ -> true
    end
  end

  @doc """
  Given a redirect response, this returns a fully qualified url to which it redirects.

  ## Examples

      iex> RecursiveClient.target_url(%HTTPoison.Response{
        body:
          ""
        headers: [
          {"Location", "https://ahrefs.com/blog/"}
        ],
        request: %HTTPoison.Request{
          body: "",
          headers: [],
          method: :get,
          options: [follow_redirect: false],
          params: %{},
          url: "https://blog.ahrefs.com"
        },
        request_url: "https://blog.ahrefs.com",
        status_code: 301
      })

      {:ok, "https://ahrefs.com/blog/"}
  """
  @spec target_url(response_struct :: HTTPoison.Response.t()) ::
          {:ok, String.t()} | {:error, any()}
  def target_url(%Response{request_url: request_url, headers: headers, status_code: status_code})
      when status_code >= 300 and status_code <= 399 do
    with {:ok, location} <- get_location(headers) do
      fully_qualify_location(request_url, location)
    end
  end

  defp fully_qualify_location(request_url, location) do
    uri1 = URI.parse(request_url)
    uri2 = URI.parse(location)

    {:ok,
     URI
     |> struct(
       Map.merge(
         uri1 |> Map.from_struct(),
         uri2
         |> Map.from_struct()
         |> Enum.reject(fn {_, v} -> is_nil(v) end)
         |> Enum.into(%{})
       )
     )
     |> URI.to_string()}
  rescue
    e in ArgumentError ->
      {:error,
       "Invalid redirect location #{inspect(location)} returned from #{request_url}; #{inspect(e)}"}
  end

  defp get_location(headers) do
    headers
    |> Enum.find_value(:error, fn
      {k, v} ->
        case String.downcase(k) do
          "location" ->
            v

          _ ->
            false
        end
    end)
    |> case do
      :error -> {:error, "Location not found"}
      location -> {:ok, location}
    end
  end
end

It’s possible some other HTTP client has more nuanced rules around its redirects that would avoid a custom solution like this for your use-case

Thanks a lot! Your idea might very much help, I thought a little about it and it might be the route I go if nothing better pops up.