kafka
logs
state
Collecting logs from Apache Kafka topics. This component is based on a modified version of rskafka which is an prue Rust, low-level and lightweight implementation.
Example
# A comma-separated list of host and port pairs that are the address
# of the Kafka brokers in a "bootstrap" Kafka cluster that a Kafka
# client connects to initially to bootstrap itself.
#
# Required
bootstrap_brokers:
- 10.14.22.123:9092
# The Kafka topics names to read events from.
#
# Regex is supported if the topic begins with `^`.
#
# Required
topics: []
# The consumer group name to be used to consume events from Kafka.
#
# Required
group: ""
# If offsets for consumer group do not exist, set them using this
# strategy.
#
# Optional
auto_offset_reset: Earliest
# The Kafka session timeout.
#
# Optional
session_timeout: 10s
# The frequency that the consumer offsets are committed(written) to
# offset storage.
#
# Optional
commit_interval: 5s
# Tell Kafka to wait until it has enough data to send before responding to the consumer.
#
# Optional
fetch_wait_max: 200ms
# The log field name to use for the Kafka message key.
#
# Optional
key_field: message_key
# The log field name to use for the Kafka topic.
#
# Optional
topic_key: topic
# The log field name to use for the Kafka partition name.
#
# Optional
partition_key: partition
# The log field name to use for the Kafka offset
#
# Optional
offset_key: offset
# The log field name to use for the Kafka headers.
#
# Optional
headers_key: headers
# Configuration for building a `Framer`.
#
# Optional
framing: bytes
# Configuration for building a `Deserializer`.
#
# Optional
decoding:
codec: json
# Determines whether or not to replace invalid UTF-8 sequences instead of failing.
#
# When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD].
#
# [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character
#
# Optional
lossy: true