Getting :gun_error {:stream_error, :protocol_error, :"Stream reset by server."}

Questions / Help
, , ,
1

:gun_error : {:stream_error, :protocol_error, :"Stream reset by server."}

In Phoenix. The stream ref is created, but somewhere before upgrade the above error presents itself. The request is never passed to &Node.Socket.init/2 Non-tls connection work fine.

Certs from GlobalSign. The same cert works fine over the web for Phx.

Any ideas??

{:gun, "~> 2.0"}
{:plug_cowboy, "~> 2.5"}

server:

  defmodule Node.Server do
      use GenServer
      
      def start_link(opts) do
        GenServer.start_link(__MODULE__, opts, name: __MODULE__)
      end
      
      def init(opts) do
        routes = [
          {:_,
           [
             {"/", Node.Socket, []}
           ]}
        ]
      
        # Compile the routes into a dispatch list
        dispatch = :cowboy_router.compile(routes)
      
        {protocol, start_fn, opts} =
          case System.get_env("IS_DOCKER") do
            "true" ->
              {:https, &:cowboy.start_tls/3,
               [
                 {:port, opts.port},
                 {:cacertfile, "/app/priv/ssl/http/ca.crt"},
                 {:certfile, "/app/priv/ssl/http/server.crt"},
                 {:keyfile, "/app/priv/ssl/http/server.key"},
               ]}
      
            _ ->
              {:http, &:cowboy.start_clear/3,
               [
                 {:port, opts.port}
               ]}
          end
      
        {:ok, _} =
          start_fn.(
            protocol,
            opts,
            %{
              env: %{dispatch: dispatch}
            }
          )
      
        {:ok, %{}}
      end
      end

client:

  defp connect_to_node({admin_domain, admin_port, port} = node, token, state) do
    admin_domain = admin_domain |> String.to_charlist()

    options =
        case System.get_env("IS_DOCKER") do
        "true" ->
          %{
            transport: :tls,
            tls_opts: [
              {:port, port},
              {:verify, :verify_peer},
              {:server_name_indication, admin_domain},
              {:customize_hostname_check, [{:match_fun, :public_key.pkix_verify_hostname_match_fun(:https)}]},
              {:cacerts, :public_key.cacerts_get()}
            ]
          }

        _ ->
          %{
            tcp_opts: [{:port, port}]
          }
       end

    {:ok, conn_pid} = :gun.open(admin_domain, admin_port, options)

    Logger.debug("Attempting to connect to Admin Node @ #{admin_domain}:#{admin_port} from local port #{port}")

    case :gun.await_up(conn_pid) do
      {:ok, _} ->
        token = token || fetch_token()

        headers = %{
          "authorization" => "#{token}"
        }

        # created fine here
        stream_ref = :gun.ws_upgrade(conn_pid, ~c"/", headers) |> IO.inspect()

        Node.Identity.set(%{
          connected_node: node
        })

        if Node.Identity.get().primary_node != node do
          :timer.send_after(900_000, self(), :connect_to_primary)
        end

        {:ok, %{state | conn_pid: conn_pid, stream_ref: stream_ref}}

      {:error, error} ->
        Logger.debug("Failed to connect to Admin Node @ #{admin_domain}:#{admin_port} from local port #{port} | Error: #{inspect(error)}")
        :gun.shutdown(conn_pid)
        {:error, state}
    end
  end

Node.Socket.inti : (again, non-tls auths fine, but this is never called when using tls)

  def init(req, _opts) do
    Logger.debug("Initiating websocket connection with request: #{inspect(req)}")

    case Map.get(req.headers, "authorization") do
      nil ->
        :cowboy_req.reply(401, req)
        {:cowboy_websocket, req, %{}}

      auth ->
        {:ok, shared_key} = Node.Identity.shared_key()

        case Node.Auth.decrypt(auth, shared_key) do
          {:ok, _decrypted} ->
            {:cowboy_websocket, req, %{}, %{idle_timeout: :infinity}}

          _ ->
            :cowboy_req.reply(401, req)
            {:cowboy_websocket, req, %{}}
        end
    end
  end

1/9/23 ~6:30 moved due to cat mistake

2

My guess would be TLS handshake failure because your cacerts don’t match. Where are you getting /app/priv/ssl/http/* from on the server?

3

GlobalSign

4

It gets past the handshake. fails at upgrade, unless I dont understand the process.

5

What if you configure the client and server to both use the GlobalSign cacert?

6

same error:

  tls_opts: [
    {:port, port},
    {:verify, :verify_peer},
    {:server_name_indication, admin_domain},
    {:customize_hostname_check, [{:match_fun, :public_key.pkix_verify_hostname_match_fun(:https)}]},
    # {:cacerts, :public_key.cacerts_get()}
    {:cacertfile, "/app/priv/ssl/http/ca.crt"}                
  ]
7

Does it help if you use a self-signed cert?

https://hexdocs.pm/phoenix/using_ssl.html#ssl-in-development

8

The same cert works fine over the web.

9

Then we’ll have to wait for some smarter experts to show up :person_shrugging:

10

No worries. TY… its just as likely you mention something that solves it. worst case someone learns something.

1 Like
11

I have used multiple sets of tls_opts at this point.

verify_none, with cacerts, with a cacertfile, etc… all of them render the same error from rst_stream_frame unless of course I dont pass the opts needed to get past handshake. In which case I can see the hand shake fail on server and client

this is the only place I can find in the code that produces the error… any idea why?

rst_stream_frame(State0, StreamID, Reason, EvHandler, EvHandlerState0) ->
	case take_stream(State0, StreamID) of
		{#stream{ref=StreamRef, reply_to=ReplyTo}, State} ->
			ReplyTo ! {gun_error, self(), stream_ref(State0, StreamRef),
				{stream_error, Reason, 'Stream reset by server.'}},
			EvHandlerState = EvHandler:cancel(#{
				stream_ref => stream_ref(State, StreamRef),
				reply_to => ReplyTo,
				endpoint => remote,
				reason => Reason
			}, EvHandlerState0),
			{{state, State}, EvHandlerState};
		error ->
			{{state, State0}, EvHandlerState0}
	end.