I am trying to connect kaffe with a Kafka image and facing this error on the app startup ,
[error] GenServer :kaffe_producer_client terminating
** (stop) [{{'kafkadocker_kafka_1', 9092}, {:nxdomain, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}]
(brod) /Tushar/POC/kaffe_oban/deps/brod/src/brod_client.erl:554: :brod_client.ensure_metadata_connection/1
(brod) /Tushar/POC/kaffe_oban/deps/brod/src/brod_client.erl:300: :brod_client.handle_info/2
(stdlib) gen_server.erl:637: :gen_server.try_dispatch/4
(stdlib) gen_server.erl:711: :gen_server.handle_msg/6
(stdlib) proc_lib.erl:249: :proc_lib.init_p_do_apply/3
Last message: :init
State: {:state, :kaffe_producer_client, [{'kafkadocker_kafka_1', 9092}], :undefined, [], :undefined, :undefined, [auto_start_producers: true, allow_topic_auto_creation: false, default_producer_config: [required_acks: -1, ack_timeout: 1000, partition_buffer_limit: 512, partition_onwire_limit: 1, max_batch_size: 1048576, max_retries: 3, retry_backoff_ms: 500, compression: :no_compression, min_compression_batch_size: 1024]], :kaffe_producer_client}
I am using 3 docker images for zookeeper , Kafka and Kafka Manager by dockerkafka ,
3132e58b1cb1 dockerkafka/kafka "/docker-entrypoint.…" 36 minutes ago Up 36 minutes 0.0.0.0:9092->9092/tcp kafkadocker_kafka_1
e1c02040e58a dockerkafka/zookeeper "/docker-entrypoint.…" 36 minutes ago Up 36 minutes 2181/tcp, 2888/tcp, 3888/tcp kafkadocker_zookeeper_1
Below is the docker inspect for the container kafkadocker_kafka_1 ,
[
{
"Id": "3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31",
"Created": "2020-06-26T11:08:24.201560847Z",
"Path": "/docker-entrypoint.sh",
"Args": [
"kafka-server-start.sh",
"/opt/kafka/config/server.properties"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 6889,
"ExitCode": 0,
"Error": "",
"StartedAt": "2020-06-26T11:08:24.611187815Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:9b40c23a50e2e28419a4980deb8875ed9cdd426d35b78be7c2d59e8d67cf30c3",
"ResolvConfPath": "/var/lib/docker/containers/3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31/hostname",
"HostsPath": "/var/lib/docker/containers/3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31/hosts",
"LogPath": "/var/lib/docker/containers/3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31/3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31-json.log",
"Name": "/kafkadocker_kafka_1",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": null,
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "default",
"PortBindings": {
"9092/tcp": [
{
"HostIp": "",
"HostPort": "9092"
}
]
},
"RestartPolicy": {
"Name": "no",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"Capabilities": null,
"Dns": [],
"DnsOptions": [],
"DnsSearch": [],
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "private",
"Cgroup": "",
"Links": [
"/kafkadocker_zookeeper_1:/kafkadocker_kafka_1/zookeeper"
],
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 0,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": [],
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DeviceCgroupRules": null,
"DeviceRequests": null,
"KernelMemory": 0,
"KernelMemoryTCP": 0,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": null,
"OomKillDisable": false,
"PidsLimit": null,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": [
"/proc/asound",
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware"
],
"ReadonlyPaths": [
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
]
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/b7e3f377d7a14e914a66ee630f27bb28d486a34c66480d410b98c7f61cb6ed55-init/diff:/var/lib/docker/overlay2/bbd51bc58da8e6c4a140226a90a4dd6e33a46db2298211271fe98733204c3b1b/diff:/var/lib/docker/overlay2/acd469a760514c75be60fe22b4635aa93c625451b1b792e96c7d8fa45fe12528/diff:/var/lib/docker/overlay2/d48a8ff7eed7c775174180381feab8543b396f160fe1d3c4ec2c9cda210b8773/diff:/var/lib/docker/overlay2/e8dcc14e2e16f360e475c30f08d27fee69b068623813761a5d8bf97a88062733/diff:/var/lib/docker/overlay2/5996722db7f2c46bc5d1e91f8c6c720d8547e58a7f34536cfaf7ea4df751edf9/diff:/var/lib/docker/overlay2/514dcff1f32060e899e8a9a67a952d55aa8373c1bb9a9a44d3686f95f19a5f96/diff:/var/lib/docker/overlay2/0ced0e041c037690a894aea2303f0d9707d0eeb302d046133b9d3f864bd2f4cd/diff:/var/lib/docker/overlay2/c3c05d011549b2d32c2eeb21dd9d898e382cd18389a1d5caeea46c692b83c56b/diff:/var/lib/docker/overlay2/5b715f765fb8be48624fedbe26762eb9ee82ed85b93a62eabad7582696235c83/diff:/var/lib/docker/overlay2/3bbcdf49421384a677dd25f18b6901e2f3abe72b384a564d5a8db0e47e90e87f/diff:/var/lib/docker/overlay2/4599f55506fae8b668d7e1d8e8aaeed089fe58a42c929eb965a6cbe5cf2cdf64/diff:/var/lib/docker/overlay2/5223f2c4905201d5b08170bf25cdeb8495986790dc51a415d4f7cd26741fb244/diff:/var/lib/docker/overlay2/616ef54bd352a1f93492beb16300cb934c272c80f3a3582da9ea92ba80871d4c/diff:/var/lib/docker/overlay2/c0d9460072722b378cd8ea026607fb06ad4212f21c63dd98821f8798d6710905/diff:/var/lib/docker/overlay2/c55210f5fe801ba2a749c462745e64f891db4b830a0bbd974043a40fd7e83288/diff:/var/lib/docker/overlay2/b32f2881d4fc45ce66224a4382f866c48cff46b0f75193bcf201464c77768e6d/diff",
"MergedDir": "/var/lib/docker/overlay2/b7e3f377d7a14e914a66ee630f27bb28d486a34c66480d410b98c7f61cb6ed55/merged",
"UpperDir": "/var/lib/docker/overlay2/b7e3f377d7a14e914a66ee630f27bb28d486a34c66480d410b98c7f61cb6ed55/diff",
"WorkDir": "/var/lib/docker/overlay2/b7e3f377d7a14e914a66ee630f27bb28d486a34c66480d410b98c7f61cb6ed55/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "volume",
"Name": "cad4107670b19d584fb8faf3811fcd02aea48b37ebc46ab371e93d84ea1f4e70",
"Source": "/var/lib/docker/volumes/cad4107670b19d584fb8faf3811fcd02aea48b37ebc46ab371e93d84ea1f4e70/_data",
"Destination": "/opt/kafka/config",
"Driver": "local",
"Mode": "",
"RW": true,
"Propagation": ""
}
],
"Config": {
"Hostname": "3132e58b1cb1",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"9092/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/opt/kafka/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"LANG=C.UTF-8",
"JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre",
"JAVA_VERSION=8u72",
"JAVA_DEBIAN_VERSION=8u72-b15-1~bpo8+1",
"CA_CERTIFICATES_JAVA_VERSION=20140324",
"SCALA_VERSION=2.11",
"KAFKA_VERSION=0.9.0.1"
],
"Cmd": [
"kafka-server-start.sh",
"/opt/kafka/config/server.properties"
],
"Image": "dockerkafka/kafka",
"Volumes": {
"/opt/kafka/config": {}
},
"WorkingDir": "",
"Entrypoint": [
"/docker-entrypoint.sh"
],
"OnBuild": null,
"Labels": {}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "9adad434062e579f9f038152f1fe64073bfd26689c117d3a42d8fe4742c81b69",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {
"9092/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "9092"
}
]
},
"SandboxKey": "/var/run/docker/netns/9adad434062e",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "4efecf9906b8f515f68bda6338f2fb31ed723237d1ccae0d789927280b2edfca",
"Gateway": "172.17.0.1",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"MacAddress": "02:42:ac:11:00:03",
"Networks": {
"bridge": {
"IPAMConfig": null,
"Links": null,
"Aliases": null,
"NetworkID": "861ec94a8597a0b5c0c7d552953ae3947ce2e4e0ed4648343b1109ae5e37869f",
"EndpointID": "4efecf9906b8f515f68bda6338f2fb31ed723237d1ccae0d789927280b2edfca",
"Gateway": "172.17.0.1",
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "02:42:ac:11:00:03",
"DriverOpts": null
}
}
}
}
]
config :kaffe,
producer: [
endpoints: [kafkadocker_kafka_1: 9092],
allow_topic_auto_creation: true,
# endpoints references [hostname: port]. Kafka is configured to run on port 9092.
# In this example, the hostname is localhost because we've started the Kafka server
# straight from our machine. However, if the server is dockerized, the hostname will
# be called whatever is specified by that container (usually "kafka")
topics: ["test_topic"], # add a list of topics you plan to produce messages to
]
Above is the config in config.exs ,
I tried both with localhost and the container name after mapping the port 9092 , during the container startup ,
Command I am using to being up the containers ,
docker run -d --name kafkadocker_zookeeper_1 dockerkafka/zookeeper;docker run -d -p 9092:9092 --name kafkadocker_kafka_1 --link kafkadocker_zookeeper_1:zookeeper dockerkafka/kafka;docker run -it --rm --name kafka-Manager --link kafkadocker_zookeeper_1:zookeeper --link kafkadocker_kafka_1:kafka -p 9000:9000 -e ZK_HOSTS=zookeeper:2181 dockerkafka/kafka-manager
It doesnt work and fails at the startup while connecting to the broker , does anyone know what could be the case here ?
