Help with HTTPoison POST

Questions / Help
7

No, it doesn’t. Here is the HTTP client module I wrote for a scraper, with HTTPoison for sending requests, and Floki for HTML parsing:

defmodule EkangScraper.HttpClient do
  alias EkangScraper.CookieStore

  def request_raw(method, url, params) do
    payload = URI.encode_query(params)
    cookies = Agent.get(CookieStore, &(&1)) || []
    try do
      response = case method do
        method when method in [:get, :delete] -> 
          real_url = url <> "?" <> payload
          apply(HTTPoison, :"#{method}!", [
            real_url, 
            %{},
            [hackney: [cookie: cookies]]])
        _ ->
          apply(HTTPoison, :"#{method}!", [
            url, 
            payload, 
            %{"Content-Type" => "application/x-www-form-urlencoded; charset=utf-8"},
            [hackney: [cookie: cookies]]])
      end

      case response.status_code do
        code when code >= 200 and code < 400 ->
          Agent.update(CookieStore, &(cookies(response) || &1))
          response.body
        code -> raise """
          Oops! #{code}
            URL: #{url}
            Method: #{method}
            Params: #{inspect params}
            Cookies: #{inspect cookies}
        """
      end
    rescue
      e in HTTPoison.Error ->
        raise """
          Oops! #{e.reason}!
            URL: #{url}
            Method: #{method}
            Params: #{inspect params}
            Cookies: #{inspect cookies}
        """
    end
  end

  def request(method, url, params) do
    request_raw(method, url, params)
    |> Floki.parse()
  end

  for method <- [:get, :post, :patch, :put, :delete] do
    def unquote(:"#{method}_raw")(url, params \\ %{}) do
      request_raw(unquote(method), url, params)
    end

    def unquote(method)(url, params \\ %{}) do
      request(unquote(method), url, params)
    end
  end

  defp cookies(%HTTPoison.Response{} = resp) do
    resp.headers
    |> Enum.filter(fn
      {"Set-Cookie", _} -> true
      _ -> false
    end)
    |> Enum.map(fn{_, cookie} -> cookie end)
  end
end

The CookieStore is just a name for an Agent process

# in application.ex
children = [
      worker(Agent, [fn -> nil end, [name: EkangScraper.CookieStore]])
]

How to use

# GET request. 
# Params are appended to URL as query string. 
# Cookies are automatically handled
doc = EkangScraper.HttpClient.get("http://www.example.com/path", %{foo: "bar", baz: "qux"})

# POST request. 
# Params are encoded as application/x-www-form-urlencoded and put to request body.
# Cookies are automatically handled.
doc = EkangScraper.HttpClient.post("http://www.example.com/path", %{foo: "bar", baz: "qux"})

# If you just want a raw response body as a string
body_string = EkangScraper.HttpClient.get_raw("http://www.example.com/path", %{foo: "bar", baz: "qux"})

There are patch, put, delete and their _raw counterpart as well.

3 Likes