7,540
edits
Changes
→Logstash consumer
https://www.elastic.co/guide/en/logstash/current/plugins-codecs-avro.html
:[[File:ClipCapIt-190402-175630.PNG]]
<source lang="C++">
version: '3.2'
services:
zookeeper:
image: confluentinc/cp-zookeeper:5.1.2
networks:
- kafka-net
ports:
- "32181:32181"
deploy:
placement:
constraints:
- node.role == worker
environment:
ZOOKEEPER_CLIENT_PORT: 32181
ZOOKEEPER_TICK_TIME: 2000
ZOOKEEPER_SYNC_LIMIT: 2
kafka:
image: confluentinc/cp-kafka:5.1.2
networks:
- kafka-net
ports:
- target: 29092
published: 29092
protocol: tcp
deploy:
placement:
constraints:
- node.role == worker
environment:
KAFKA_ZOOKEEPER_CONNECT: "zookeeper:32181"
KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:29092"
KAFKA_BROKER_ID: 2
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
schemaregistry:
image: confluentinc/cp-schema-registry:5.1.2
networks:
- kafka-net
ports:
- target: 8081
published: 8081
protocol: tcp
deploy:
placement:
constraints:
- node.role == worker
environment:
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: "zookeeper:32181"
SCHEMA_REGISTRY_HOST_NAME: "schemaregistry"
SCHEMA_REGISTRY_DEBUG: "true"
logstash:
image: docker.elastic.co/logstash/logstash:6.6.2
networks:
- kafka-net
ports:
- "51415:51415"
environment:
LOGSPOUT: "ignore"
XPACK_MONITORING_ENABLED: "false"
volumes:
- "logstash-conf:/usr/share/logstash/pipeline"
deploy:
placement:
constraints:
- node.role == worker
restart_policy:
condition: on-failure
resources:
reservations:
memory: 100m
networks:
kafka-net:
driver: overlay
volumes:
logstash-conf:
driver: nfs
driver_opts:
share: 192.168.42.1:/home/adam/dockerStore/logstash/config/
</source>
/usr/share/logstash/pipeline/'''logstash.conf'''
<pre>
input {
kafka {
bootstrap_servers => "kafka:29092"
topics => "test-topic"
codec => avro {
schema_uri => "/usr/share/logstash/pipeline/employee.avsc"
}
}
}
output {
stdout {
codec => rubydebug
}
}
</pre>
