ExAws: HTTP ERROR: :nxdomain for URL: s3.amazonaws.com/logo/original-xx.png ATTEMPT: 10

I have an application in Phoenix 1.6.2 that uploads files to Amazon S3 using ex_aws_s3.

On localhost the upload works fine for the bucket, but in production the upload doesn’t work.

This error happens:

15:49:33.135 [warn] ExAws: HTTP ERROR: :nxdomain for URL: "https:/prd-envixo-bucket.s3.amazonaws.com/1/logo/original-549bac34-7c85-42e6-b2cd-cb3ce7f71fab.png" ATTEMPT: 10
15:49:33.136 [error] GenServer #PID<0.2635.0> terminating
** (ExAws.Error) ExAws Request Error!

{:error, :nxdomain}

    (ex_aws 2.2.9) lib/ex_aws.ex:87: ExAws.request!/2
    (my_app 0.1.0) lib/my_app_web/live/register_wizard/register_wizard_step2_live.ex:87: anonymous fn/4 in MyAppWeb.RegisterWizardStep2Live.consume_logo/2
    (phoenix_live_view 0.17.2) lib/phoenix_live_view/upload_channel.ex:20: Phoenix.LiveView.UploadChannel.consume/3

I’ve tried several different settings, but nothing works.

Looks like the ex_aws_s3 has a bug and generates the wrong URL.

My prod.secret.exs file:

config :ex_aws,
  bucket: System.fetch_env!("AWS_BUCKET"),
  cloud_front: System.fetch_env!("AWS_CLOUD_FRONT"),
  region: System.fetch_env!("AWS_REGION"),
  access_key_id: [System.fetch_env!("AWS_ACCESS_KEY_ID"), :instance_role],
  secret_access_key: [System.fetch_env!("AWS_SECRET_ACCESS_KEY"), :instance_role]

config :ex_aws, :s3, scheme: "https://"

And this is my code that uploads the file:

  @s3_bucket Application.fetch_env!(:ex_aws, :bucket)
  defp s3_url, do: "#{@s3_bucket}.s3.amazonaws.com"

  def consume_logo(socket, current_enterprise) do
    consume_uploaded_entries(socket, :logo, fn meta, entry ->
      # Delete old file if exists on S3
      if current_enterprise.logo_original_url do
        logo_path = file_s3_path(current_enterprise.logo_original_url)
        ExAws.S3.delete_object(s3_url(), logo_path) |> ExAws.request!()
      end

      if current_enterprise.logo_thumbnail_url do
        logo_path = file_s3_path(current_enterprise.logo_thumbnail_url)
        ExAws.S3.delete_object(s3_url(), logo_path) |> ExAws.request!()
      end

      # Upload new original file to S3
      original_url = Path.join(s3_url(), filename(entry, socket, :original))
      {:ok, original_contents} = File.read(meta.path)
      ExAws.S3.put_object(s3_url(), file_s3_path(original_url), original_contents) |> ExAws.request!()

      # Resize and upload thumbnail file to S3
      thumbnail_url = Path.join(s3_url(), filename(entry, socket, :thumbnail))
      thumbnail = Image.resize(meta.path, 189, 50)
      {:ok, thumbnail_contents} = File.read(thumbnail.path)
      ExAws.S3.put_object(s3_url(), file_s3_path(thumbnail_url), thumbnail_contents) |> ExAws.request!()

      # Delete temp files
      File.rm!(meta.path)
      File.rm!(thumbnail.path)

      # Return a new url image for original and thumbnail
      {original_url, thumbnail_url}
    end)
  end
  
  defp file_s3_path(file_url) do
    file_url |> String.replace("#{s3_url()}/", "")
  end

  defp filename(entry, socket, type) do
    [ext | _] = MIME.extensions(entry.client_type)
    "#{socket.assigns.current_enterprise.id}/logo/#{type}-#{entry.uuid}.#{ext}"
  end

Does anyone know how I can solve this problem?

Thank you for your help.

https:/prd-envixo... That lone / seems very sus.

The code in ex_aws_s3 that builds a URL doesn’t add/remove any slashes from the configured value of scheme.

2 Likes

It’s very weird. I didn’t understand why this behavior.

Where is this config coming from? This isn’t part of the ExAws config at all.

I’m using docker + kubernetes.

You’re right, I made some changes to default of ex_aws_s3.

Now my prod.secret.exs looks like this:

use Mix.Config

# Ex AWS
config :ex_aws,
  region: System.fetch_env!("AWS_REGION"),
  access_key_id: [System.fetch_env!("AWS_ACCESS_KEY_ID"), :instance_role],
  secret_access_key: [System.fetch_env!("AWS_SECRET_ACCESS_KEY"), :instance_role]

# App configs
config :my_app,
  bucket: System.fetch_env!("AWS_BUCKET"),
  cloud_front: System.fetch_env!("AWS_CLOUD_FRONT")

# ## Using releases (Elixir v1.9+)
#
# If you are doing OTP releases, you need to instruct Phoenix
# to start each relevant endpoint:
#
config :my_app, MyAppWeb.Endpoint, server: true
#
# Then you can assemble a release by calling `mix release`.
# See `mix help release` for more information.

And I changed my upload code to:

 @s3_bucket Application.fetch_env!(:my_app, :bucket)
  defp s3_url, do: "//#{@s3_bucket}.s3.amazonaws.com"

  def consume_logo(socket, current_enterprise) do
    consume_uploaded_entries(socket, :logo, fn meta, entry ->
      # Delete old file if exists on S3
      if current_enterprise.logo_original_url do
        logo_path = file_s3_path(current_enterprise.logo_original_url)
        ExAws.S3.delete_object(@s3_bucket, logo_path) |> ExAws.request!()
      end

      if current_enterprise.logo_thumbnail_url do
        logo_path = file_s3_path(current_enterprise.logo_thumbnail_url)
        ExAws.S3.delete_object(@s3_bucket, logo_path) |> ExAws.request!()
      end

      # Upload new original file to S3
      original_url = Path.join(s3_url(), filename(entry, socket, :original))
      {:ok, original_contents} = File.read(meta.path)
      ExAws.S3.put_object(@s3_bucket, file_s3_path(original_url), original_contents) |> ExAws.request!()

      # Resize and upload thumbnail file to S3
      thumbnail_url = Path.join(s3_url(), filename(entry, socket, :thumbnail))
      thumbnail = Image.resize(meta.path, 189, 50)
      {:ok, thumbnail_contents} = File.read(thumbnail.path)
      ExAws.S3.put_object(@s3_bucket, file_s3_path(thumbnail_url), thumbnail_contents) |> ExAws.request!()

      # Delete temp files
      File.rm!(meta.path)
      File.rm!(thumbnail.path)

      # Return a new url image for original and thumbnail
      {original_url, thumbnail_url}
    end)
  end

  defp file_s3_path(file_url) do
    file_url |> String.replace("#{s3_url()}/", "")
  end

  defp filename(entry, socket, type) do
    [ext | _] = MIME.extensions(entry.client_type)
    "#{socket.assigns.current_enterprise.id}/logo/#{type}-#{entry.uuid}.#{ext}"
  end

But the error persists:

22:23:11.708 [warn] ExAws: HTTP ERROR: :nxdomain for URL: "https:/prd-envixo-bucket/1/logo/original-65aa0bfe-c304-41c5-bb9b-97550b9f2d1c.png" ATTEMPT: 1
22:23:11.728 [warn] ExAws: HTTP ERROR: :nxdomain for URL: "https:/prd-envixo-bucket/1/logo/original-65aa0bfe-c304-41c5-bb9b-97550b9f2d1c.png" ATTEMPT: 2
22:23:11.751 [warn] ExAws: HTTP ERROR: :nxdomain for URL: "https:/prd-envixo-bucket/1/logo/original-65aa0bfe-c304-41c5-bb9b-97550b9f2d1c.png" ATTEMPT: 3
22:23:11.813 [warn] ExAws: HTTP ERROR: :nxdomain for URL: "https:/prd-envixo-bucket/1/logo/original-65aa0bfe-c304-41c5-bb9b-97550b9f2d1c.png" ATTEMPT: 4
22:23:11.931 [warn] ExAws: HTTP ERROR: :nxdomain for URL: "https:/prd-envixo-bucket/1/logo/original-65aa0bfe-c304-41c5-bb9b-97550b9f2d1c.png" ATTEMPT: 5
22:23:12.168 [warn] ExAws: HTTP ERROR: :nxdomain for URL: "https:/prd-envixo-bucket/1/logo/original-65aa0bfe-c304-41c5-bb9b-97550b9f2d1c.png" ATTEMPT: 6
22:23:12.398 [warn] ExAws: HTTP ERROR: :nxdomain for URL: "https:/prd-envixo-bucket/1/logo/original-65aa0bfe-c304-41c5-bb9b-97550b9f2d1c.png" ATTEMPT: 7
22:23:13.429 [warn] ExAws: HTTP ERROR: :nxdomain for URL: "https:/prd-envixo-bucket/1/logo/original-65aa0bfe-c304-41c5-bb9b-97550b9f2d1c.png" ATTEMPT: 8
22:23:14.848 [warn] ExAws: HTTP ERROR: :nxdomain for URL: "https:/prd-envixo-bucket/1/logo/original-65aa0bfe-c304-41c5-bb9b-97550b9f2d1c.png" ATTEMPT: 9
22:23:15.140 [warn] ExAws: HTTP ERROR: :nxdomain for URL: "https:/prd-envixo-bucket/1/logo/original-65aa0bfe-c304-41c5-bb9b-97550b9f2d1c.png" ATTEMPT: 10
22:23:15.140 [error] GenServer #PID<0.2656.0> terminating
** (ExAws.Error) ExAws Request Error!

{:error, :nxdomain}

    (ex_aws 2.2.9) lib/ex_aws.ex:87: ExAws.request!/2
    (my_app 0.1.0) lib/my_app_web/live/register_wizard/register_wizard_step2_live.ex:87: anonymous fn/4 in MyAppWeb.RegisterWizardStep2Live.consume_logo/2
    (phoenix_live_view 0.17.2) lib/phoenix_live_view/upload_channel.ex:20: Phoenix.LiveView.UploadChannel.consume/3
    (elixir 1.12.3) lib/enum.ex:1582: Enum."-map/2-lists^map/1-0-"/2
    (my_app 0.1.0) lib/my_app_web/live/register_wizard/register_wizard_step2_live.ex:32: MyAppWeb.RegisterWizardStep2Live.handle_event/3
    (phoenix_live_view 0.17.2) lib/phoenix_live_view/channel.ex:349: anonymous fn/3 in Phoenix.LiveView.Channel.view_handle_event/3
    (telemetry 1.0.0) /app/deps/telemetry/src/telemetry.erl:293: :telemetry.span/3
    (phoenix_live_view 0.17.2) lib/phoenix_live_view/channel.ex:206: Phoenix.LiveView.Channel.handle_info/2

Any other ideas?

Thank you for your help.

This is just straight up malformed. I think your code here is doing the substitution wrong:

  defp file_s3_path(file_url) do
    file_url |> String.replace("#{s3_url()}/", "")
  end

Basically though, the bug is in your code, not the ExAws code base here.

I made some code changes.

And I added a log to understand the value of the variables, but it looks like the code is working correctly.

  @s3_bucket Application.fetch_env!(:my_app, :bucket)
  defp s3_url, do: "//#{@s3_bucket}.s3.amazonaws.com"

  def consume_logo(socket, current_enterprise) do
    consume_uploaded_entries(socket, :logo, fn meta, entry ->
      # Delete old file if exists on S3
      if current_enterprise.logo_original_url do
        logo_path = file_s3_path(current_enterprise.logo_original_url)
        ExAws.S3.delete_object(@s3_bucket, logo_path) |> ExAws.request!()
      end

      if current_enterprise.logo_thumbnail_url do
        logo_path = file_s3_path(current_enterprise.logo_thumbnail_url)
        ExAws.S3.delete_object(@s3_bucket, logo_path) |> ExAws.request!()
      end

      # Upload new original file to S3
      original_url = Path.join(s3_url(), filename(entry, socket, :original))
      {:ok, original_contents} = File.read(meta.path)

      # Honeybadger notify
      Honeybadger.notify("Debug ex_aws_s3 put_object", metadata: %{s3_url: s3_url(), filename: filename(entry, socket, :original), original_url: original_url, s3_bucket: @s3_bucket, file_s3_path: file_s3_path(original_url)})

      ExAws.S3.put_object(@s3_bucket, file_s3_path(original_url), original_contents) |> ExAws.request!()

The variables values:

{
  "file_s3_path" : "/1/logo/original-6bfd3d5b-6926-4c9a-99fa-ce9819c09ad4.png",
  "filename" : "/1/logo/original-6bfd3d5b-6926-4c9a-99fa-ce9819c09ad4.png",
  "original_url" : "//prd-envixo-bucket.s3.amazonaws.com/1/logo/original-6bfd3d5b-6926-4c9a-99fa-ce9819c09ad4.png",
  "s3_bucket" : "prd-envixo-bucket",
  "s3_url" : "//prd-envixo-bucket.s3.amazonaws.com"
}

Any other ideas?

Thank you for your help.