Welcome to the forum @spurgus!

If the response includes the content-length header then you can check that from a response step:

req =
  Req.new()
  |> Req.Request.prepend_response_steps(validate_content_length: fn {req, resp} ->
    with [header] <- Req.Response.get_header(resp, "content-length"),
         {content_length, ""} <- Integer.parse(header) do
      if content_length > @max_content_length do
        Req.cancel_async_response(resp)
        {req, RuntimeError.exception(message: "content-length too large")}
      else
        {req, resp}
      end
    else
      _ ->
        Req.cancel_async_response(resp)
        {req, RuntimeError.exception(message: "Invalid content-length")}
    end
  end)

Otherwise, you’ll have to keep track of received bytes and halt the request. You mentioned streaming but didn’t say which form of streaming (into: :self, into: &fun/2, into: collectable). Here’s an example for the function form of streaming:

Req.get(req, into: fn {:data, data}, {req, resp} ->
   resp = Req.Response.update_private(resp, :length, 0, & &1 + byte_size(data))

   if Req.Response.get_private(resp, :length) > @max_content_length do
     {:halt, {req, RuntimeError.exception(message: "content length too large")}}
   else
     {:cont, {req, resp}}
   end
 end)

Thanks a lot for your help @jswanner !

Most of the times the content-length header will be present but I think that checking for the actual size while downloading is a good measure just in case it’s missing or wrong.

This is what I’m trying, but I get an empty body in the response??

defmodule Utils.Downloader do
  require Logger

  @default_receive_timeout_ms 20_000
  @default_max_size_bytes 20 * 1_024 * 1_024

  def download(url, opts \\ []) do
    with {:ok, %Req.Response{status: 200, body: body, headers: headers}} <-
           request(url, opts),
         {:ok, content_type} <- get_content_type(headers) do
      Logger.debug("[#{__MODULE__}] finished downloading...")

      {:ok, %{content_type: content_type, data: body}}
    else
      {:error, :cant_get_content_type} ->
        {:error, :cant_get_content_type}

      %Req.Response{status: status} when status != 200 ->
        {:error, :cant_download}
    end
  end

  defp request(url, opts \\ []) do
    Logger.debug("[#{__MODULE__}] downloading...")

    receive_timeout_ms = opts[:receive_timeout_ms] || @default_receive_timeout_ms
    max_size_bytes = opts[:max_size_bytes] || @default_max_size_bytes

    Req.new()
    |> Req.get(
      url: url,
      receive_timeout: receive_timeout_ms,
      into: fn {:data, data}, {req, resp} ->
        resp = Req.Response.update_private(resp, :length, 0, &(&1 + byte_size(data)))

        if Req.Response.get_private(resp, :length) > @max_content_length do
          {:halt, {req, RuntimeError.exception(message: "streamed content length too large")}}
        else
          {:cont, {req, resp}}
        end
      end
    )
    |> IO.inspect(label: "request finished")
  end

  defp get_content_type(headers) do
    headers
    |> Enum.find(fn {key, _} -> String.downcase(key) == "content-type" end)
    |> case do
      {_, value} when is_list(value) -> {:ok, hd(value)}
      {_, value} when is_binary(value) -> {:ok, value}
      nil -> {:error, :cant_get_content_type}
    end
  end
end

Try:

{:cont, {req, update_in(resp.body, &(&1 <> data))}}

I believe this into: &fun/2 option is envisioned for scenarios where you’ll be doing something with the data as it’s coming in (such as sending it to another process), rather than accumulating it and processing it at the end.