@ragamuf and @wojtekmach I ended up implementing a small and lightweight B2 client using the advice in the article. This is only for my own use, so there is room for improvement (feel free to add any suggestions/feedback!) but here is what the end result looks like:

# in Livebook (or your `mix.exs`)
Mix.install([
  {:req, "~> 0.5.6"},
  {:typed_struct, "~> 0.3.0"}
])

defmodule B2.Client do
  use TypedStruct

  typedstruct do
    field :app_key_id, String.t(), enforce: true
    field :app_key, String.t(), enforce: true
    field :endpoint_url, String.t(), enforce: true
    field :bucket, String.t(), enforce: true
  end
end

defmodule B2 do
  defp new(%B2.Client{} = client, options) when is_list(options) do
    Req.new(
      # https://hexdocs.pm/req/Req.Steps.html#put_aws_sigv4/1
      aws_sigv4: [
        service: :s3,
        access_key_id: client.app_key_id,
        secret_access_key: client.app_key
      ],
      base_url: "#{client.endpoint_url}/#{client.bucket}"
      # retry: :transient
    )
    |> Req.merge(options)
  end

  def client(app_key_id, app_key, endpoint_url, bucket) do
    %B2.Client{
      app_key_id: app_key_id,
      app_key: app_key,
      endpoint_url: endpoint_url,
      bucket: bucket
    }
  end

  def get_object(%B2.Client{} = client, key, options \\ []) when is_binary(key) do
    req = new(client, options)
      
    case Req.get(req, url: key) do
      {:ok, %{status: 200, body: body}} -> body
      {:ok, %{status: 404}} -> nil
      error -> error
    end
  end

  def put_object(%B2.Client{} = client, key, object, options \\ [])
    when is_binary(key) and is_binary(object) do
    req =
      new(client, options)
      # TODO: properly set the content MIME type
      # https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
      # |> Req.Request.put_header("Content-Type", "text/csv")
    
    case Req.put(req, url: key, body: object) do
      {:ok, %{status: 200}} -> :ok
      {:ok, %{status: n} = resp} when n > 300 -> {:error, resp}
      error -> error
    end
  end
end

And here is some example usage:

# configure the client using the Application environment or environment variables etc

# in this example I am using LiveBook secrets
b2_app_key_id = System.get_env("LB_B2_APP_KEY_ID")
b2_app_key = System.get_env("LB_B2_APP_KEY")

# make sure to replace your <region>
endpoint_url = "https://s3.<region>.backblazeb2.com"
# make sure to replace your <bucket>
bucket = "<bucket>"

client = B2.client(b2_app_key_id, b2_app_key, endpoint_url, bucket)

# write a file
contents = """
col_1,col_2,col_3
A1,B1,C1
A2,B2,C2
A3,B3,C3
A4,B4,C3
"""
:ok = client |> B2.put_object("example.csv", contents)

# read a file
object = client |> B2.get_object("example.csv")

Thanks again to everyone that provided recommendations, I really appreciate the feedback