tushar
Kaffe connection with dockerized kafka
I am trying to connect kaffe with a Kafka image and facing this error on the app startup ,
[error] GenServer :kaffe_producer_client terminating
** (stop) [{{'kafkadocker_kafka_1', 9092}, {:nxdomain, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}]
(brod) /Tushar/POC/kaffe_oban/deps/brod/src/brod_client.erl:554: :brod_client.ensure_metadata_connection/1
(brod) /Tushar/POC/kaffe_oban/deps/brod/src/brod_client.erl:300: :brod_client.handle_info/2
(stdlib) gen_server.erl:637: :gen_server.try_dispatch/4
(stdlib) gen_server.erl:711: :gen_server.handle_msg/6
(stdlib) proc_lib.erl:249: :proc_lib.init_p_do_apply/3
Last message: :init
State: {:state, :kaffe_producer_client, [{'kafkadocker_kafka_1', 9092}], :undefined, [], :undefined, :undefined, [auto_start_producers: true, allow_topic_auto_creation: false, default_producer_config: [required_acks: -1, ack_timeout: 1000, partition_buffer_limit: 512, partition_onwire_limit: 1, max_batch_size: 1048576, max_retries: 3, retry_backoff_ms: 500, compression: :no_compression, min_compression_batch_size: 1024]], :kaffe_producer_client}
I am using 3 docker images for zookeeper , Kafka and Kafka Manager by dockerkafka ,
3132e58b1cb1 dockerkafka/kafka "/docker-entrypoint.…" 36 minutes ago Up 36 minutes 0.0.0.0:9092->9092/tcp kafkadocker_kafka_1
e1c02040e58a dockerkafka/zookeeper "/docker-entrypoint.…" 36 minutes ago Up 36 minutes 2181/tcp, 2888/tcp, 3888/tcp kafkadocker_zookeeper_1
Below is the docker inspect for the container kafkadocker_kafka_1 ,
[
{
"Id": "3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31",
"Created": "2020-06-26T11:08:24.201560847Z",
"Path": "/docker-entrypoint.sh",
"Args": [
"kafka-server-start.sh",
"/opt/kafka/config/server.properties"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 6889,
"ExitCode": 0,
"Error": "",
"StartedAt": "2020-06-26T11:08:24.611187815Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:9b40c23a50e2e28419a4980deb8875ed9cdd426d35b78be7c2d59e8d67cf30c3",
"ResolvConfPath": "/var/lib/docker/containers/3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31/hostname",
"HostsPath": "/var/lib/docker/containers/3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31/hosts",
"LogPath": "/var/lib/docker/containers/3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31/3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31-json.log",
"Name": "/kafkadocker_kafka_1",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": null,
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "default",
"PortBindings": {
"9092/tcp": [
{
"HostIp": "",
"HostPort": "9092"
}
]
},
"RestartPolicy": {
"Name": "no",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"Capabilities": null,
"Dns": [],
"DnsOptions": [],
"DnsSearch": [],
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "private",
"Cgroup": "",
"Links": [
"/kafkadocker_zookeeper_1:/kafkadocker_kafka_1/zookeeper"
],
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 0,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": [],
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DeviceCgroupRules": null,
"DeviceRequests": null,
"KernelMemory": 0,
"KernelMemoryTCP": 0,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": null,
"OomKillDisable": false,
"PidsLimit": null,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": [
"/proc/asound",
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware"
],
"ReadonlyPaths": [
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
]
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/b7e3f377d7a14e914a66ee630f27bb28d486a34c66480d410b98c7f61cb6ed55-init/diff:/var/lib/docker/overlay2/bbd51bc58da8e6c4a140226a90a4dd6e33a46db2298211271fe98733204c3b1b/diff:/var/lib/docker/overlay2/acd469a760514c75be60fe22b4635aa93c625451b1b792e96c7d8fa45fe12528/diff:/var/lib/docker/overlay2/d48a8ff7eed7c775174180381feab8543b396f160fe1d3c4ec2c9cda210b8773/diff:/var/lib/docker/overlay2/e8dcc14e2e16f360e475c30f08d27fee69b068623813761a5d8bf97a88062733/diff:/var/lib/docker/overlay2/5996722db7f2c46bc5d1e91f8c6c720d8547e58a7f34536cfaf7ea4df751edf9/diff:/var/lib/docker/overlay2/514dcff1f32060e899e8a9a67a952d55aa8373c1bb9a9a44d3686f95f19a5f96/diff:/var/lib/docker/overlay2/0ced0e041c037690a894aea2303f0d9707d0eeb302d046133b9d3f864bd2f4cd/diff:/var/lib/docker/overlay2/c3c05d011549b2d32c2eeb21dd9d898e382cd18389a1d5caeea46c692b83c56b/diff:/var/lib/docker/overlay2/5b715f765fb8be48624fedbe26762eb9ee82ed85b93a62eabad7582696235c83/diff:/var/lib/docker/overlay2/3bbcdf49421384a677dd25f18b6901e2f3abe72b384a564d5a8db0e47e90e87f/diff:/var/lib/docker/overlay2/4599f55506fae8b668d7e1d8e8aaeed089fe58a42c929eb965a6cbe5cf2cdf64/diff:/var/lib/docker/overlay2/5223f2c4905201d5b08170bf25cdeb8495986790dc51a415d4f7cd26741fb244/diff:/var/lib/docker/overlay2/616ef54bd352a1f93492beb16300cb934c272c80f3a3582da9ea92ba80871d4c/diff:/var/lib/docker/overlay2/c0d9460072722b378cd8ea026607fb06ad4212f21c63dd98821f8798d6710905/diff:/var/lib/docker/overlay2/c55210f5fe801ba2a749c462745e64f891db4b830a0bbd974043a40fd7e83288/diff:/var/lib/docker/overlay2/b32f2881d4fc45ce66224a4382f866c48cff46b0f75193bcf201464c77768e6d/diff",
"MergedDir": "/var/lib/docker/overlay2/b7e3f377d7a14e914a66ee630f27bb28d486a34c66480d410b98c7f61cb6ed55/merged",
"UpperDir": "/var/lib/docker/overlay2/b7e3f377d7a14e914a66ee630f27bb28d486a34c66480d410b98c7f61cb6ed55/diff",
"WorkDir": "/var/lib/docker/overlay2/b7e3f377d7a14e914a66ee630f27bb28d486a34c66480d410b98c7f61cb6ed55/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "volume",
"Name": "cad4107670b19d584fb8faf3811fcd02aea48b37ebc46ab371e93d84ea1f4e70",
"Source": "/var/lib/docker/volumes/cad4107670b19d584fb8faf3811fcd02aea48b37ebc46ab371e93d84ea1f4e70/_data",
"Destination": "/opt/kafka/config",
"Driver": "local",
"Mode": "",
"RW": true,
"Propagation": ""
}
],
"Config": {
"Hostname": "3132e58b1cb1",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"9092/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/opt/kafka/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"LANG=C.UTF-8",
"JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre",
"JAVA_VERSION=8u72",
"JAVA_DEBIAN_VERSION=8u72-b15-1~bpo8+1",
"CA_CERTIFICATES_JAVA_VERSION=20140324",
"SCALA_VERSION=2.11",
"KAFKA_VERSION=0.9.0.1"
],
"Cmd": [
"kafka-server-start.sh",
"/opt/kafka/config/server.properties"
],
"Image": "dockerkafka/kafka",
"Volumes": {
"/opt/kafka/config": {}
},
"WorkingDir": "",
"Entrypoint": [
"/docker-entrypoint.sh"
],
"OnBuild": null,
"Labels": {}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "9adad434062e579f9f038152f1fe64073bfd26689c117d3a42d8fe4742c81b69",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {
"9092/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "9092"
}
]
},
"SandboxKey": "/var/run/docker/netns/9adad434062e",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "4efecf9906b8f515f68bda6338f2fb31ed723237d1ccae0d789927280b2edfca",
"Gateway": "172.17.0.1",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"MacAddress": "02:42:ac:11:00:03",
"Networks": {
"bridge": {
"IPAMConfig": null,
"Links": null,
"Aliases": null,
"NetworkID": "861ec94a8597a0b5c0c7d552953ae3947ce2e4e0ed4648343b1109ae5e37869f",
"EndpointID": "4efecf9906b8f515f68bda6338f2fb31ed723237d1ccae0d789927280b2edfca",
"Gateway": "172.17.0.1",
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "02:42:ac:11:00:03",
"DriverOpts": null
}
}
}
}
]
config :kaffe,
producer: [
endpoints: [kafkadocker_kafka_1: 9092],
allow_topic_auto_creation: true,
# endpoints references [hostname: port]. Kafka is configured to run on port 9092.
# In this example, the hostname is localhost because we've started the Kafka server
# straight from our machine. However, if the server is dockerized, the hostname will
# be called whatever is specified by that container (usually "kafka")
topics: ["test_topic"], # add a list of topics you plan to produce messages to
]
Above is the config in config.exs ,
I tried both with localhost and the container name after mapping the port 9092 , during the container startup ,
Command I am using to being up the containers ,
docker run -d --name kafkadocker_zookeeper_1 dockerkafka/zookeeper;docker run -d -p 9092:9092 --name kafkadocker_kafka_1 --link kafkadocker_zookeeper_1:zookeeper dockerkafka/kafka;docker run -it --rm --name kafka-Manager --link kafkadocker_zookeeper_1:zookeeper --link kafkadocker_kafka_1:kafka -p 9000:9000 -e ZK_HOSTS=zookeeper:2181 dockerkafka/kafka-manager
It doesnt work and fails at the startup while connecting to the broker , does anyone know what could be the case here ?
First Post!
Popular in Questions
Hi All,
I set a environment variables in dev.exs , like below code.
when i start server, how can i set the ${enable} value?
thanks.
d...
New
Hello, can anybody help here..? I have a list of players and I what to delete an element, but every for loop the list is reverting to ori...
New
I am constructing a JSON object (map) and I need to conditionally set a field. I’m trying to write proper elixir-way code… and I’m at a l...
New
Hello, how can I check the Phoenix version ?
Thanks !
New
How to bind a phoenix app to a specific ip address?
could not find anything about that, nowhere, unfortunately, but for me this is quite...
New
Is there a way to rollback a specific migration and only that one (“skipping” all the other ones)?
Would
mix ecto.rollback -v 200809061...
New
Why is it that the mnesia database isn’t the most preferred database for use in Elixir/Phoenix?
New
I will often find my self writing things similar to:
case some_value do
nil -> something()
"" -> something()
_ -> somethi...
New
I’m not a pro in using Regex and can’t figure out why the following behaviour happens, especially if we take into account the difference ...
New
I had some trouble figuring out how to make many-to-many associations work. Once I got it working, I wrote a blog post. Because I’m a nov...
New
Other popular topics
Hello everyone,
I try to use an Javascript Event Handler in my root.html.leex file.
Therefore I created a function in the app.js file: ...
New
I believe there are people here who are dealing with CSV files import on the daily basis, and since Excel is a really popular tool there ...
New
Hi everyone,
One of the features added to Elixir early on to help integration with Erlang code was the idea of overridable function defi...
New
In templates/appointment/index.html.eex:
<%= for appointment <- @appointments do %>
<tr>
<td><%= appoi...
New
I want to highlight html closing tags when i click a html tag. That works in .html files but doesnt work for html.eex templates. How can...
New
I am VERY much an elixir newbie. I have taken one elixir course and one phoenix course on Udemy. During that course, I saw the instructor...
New
I am trying to start a new phoenix project with elixir 1.9, but mix phx.new does not work. It says that ** (Mix) The task "phx.new" could...
New
We’ve put together this wiki for Phoenix LiveView - please feel free to add any info you feel is worth including.
What is Phoenix LiveV...
New
There are pre-rolled solutions for other frameworks that do work. However, Phoenix does not seem to have these. Have people had good expe...
New
Lets say I have map like this fetching from my database
%{"_id" => #BSON.ObjectId<58eb1a7a9ad169198c3dXXXX>, "email" => ...
New
Categories:
Sub Categories:
Forums
Popular Tags
- #ecto
- #liveview
- #troubleshooting
- #learning-elixir
- #deployment
- #library
- #erlang
- #testing
- #genserver
- #mix
- #absinthe
- #remote-other
- #otp
- #plug
- #how-to-question
- #macros
- #postgres
- #channels
- #elixirconf
- #exunit
- #discussion
- #code-sync
- #javascript
- #podcasts
- #onsite
- #dialyzer
- #docker
- #authentication
- #umbrella
- #full-time-contract
- #podcasts-by-brainlid
- #ecto-query
- #elixir-ls
- #phoenix_html
- #iex
- #blog-post
- #graphql
- #genstage
- #ai
- #websockets
- #supervisor
- #advent-of-code
- #elixirconf-us
- #distillery
- #processes
- #forms
- #api
- #metaprogramming
- #security
- #performance








