Does this not work?
into: fn {:data, data}, {req, resp} ->
resp = Req.Response.update_private(resp, :length, byte_size(data), &(&1 + byte_size(data)))
if Req.Response.get_private(resp, :length) > max_size_bytes do
{:halt, {req, RuntimeError.exception(message: "streamed content length too large")}}
else
{:cont, {req, update_in(resp.body, &(&1 <> data))}}
end
end
Thanks, everyone, for your help, you’re awesome!
For the record, here’s the full code:
defmodule Downloader do
require Logger
@default_receive_timeout_ms 10_000
@default_max_retries 3
@default_max_size_bytes 20 * 1_024 * 1_024
def download(url, opts \\ []) do
Logger.debug("[#{__MODULE__}] downloading...")
with {:ok, :valid_url} <- valid_url?(url),
{:ok, req_client} <- prepare_req_client(opts),
{:ok, %Req.Response{status: 200, body: body, headers: headers}} <-
Req.get(req_client, url: url),
{:ok, content_type} <- get_header(headers, "content-type") do
Logger.debug("[#{__MODULE__}] finished downloading...")
{:ok, %{data: body, content_type: content_type}}
else
{:error, :invalid_url} ->
{:error, :invalid_url}
{:ok, %Req.Response{status: status}} ->
{:error, :remote_server_error, status}
{:error, %Req.TransportError{reason: :econnrefused}} ->
{:error, :remote_server_error, :econnrefused}
{:error, %Req.TransportError{reason: :timeout}} ->
{:error, :remote_server_error, :timeout}
{:error, %RuntimeError{message: "streamed content length too large"}} ->
{:error, :max_size_exceeded}
end
end
defp valid_url?(url) when is_binary(url) do
case URI.parse(url) do
%URI{scheme: scheme, host: host} when scheme in ["http", "https"] and is_binary(host) ->
{:ok, :valid_url}
_ ->
{:error, :invalid_url}
end
end
defp valid_url?(_) do
{:error, :invalid_url}
end
defp get_header(headers, name) do
headers
|> Enum.find(fn {key, _} -> String.downcase(key) == name end)
|> case do
{_, value} when is_list(value) -> {:ok, hd(value)}
{_, value} when is_binary(value) -> {:ok, value}
nil -> {:ok, nil}
end
end
defp prepare_req_client(opts) do
receive_timeout_ms = opts[:receive_timeout_ms] || @default_receive_timeout_ms
max_retries = opts[:max_retries] || @default_max_retries
max_size_bytes = opts[:max_size_bytes] || @default_max_size_bytes
client =
Req.new(
receive_timeout: receive_timeout_ms,
max_retries: max_retries,
into: fn {:data, data}, {req, resp} ->
resp =
Req.Response.update_private(
resp,
:length,
byte_size(data),
&(&1 + byte_size(data))
)
if Req.Response.get_private(resp, :length) > max_size_bytes do
{:halt, {req, RuntimeError.exception(message: "streamed content length too large")}}
else
{:cont, {req, update_in(resp.body, &(&1 <> data))}}
end
end
)
{:ok, client}
end
end