You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
281 lines
7.5 KiB
281 lines
7.5 KiB
3 years ago
|
package kafka
|
||
|
|
||
|
/**
|
||
|
* Copyright 2016 Confluent Inc.
|
||
|
*
|
||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
* you may not use this file except in compliance with the License.
|
||
|
* You may obtain a copy of the License at
|
||
|
*
|
||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||
|
*
|
||
|
* Unless required by applicable law or agreed to in writing, software
|
||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
* See the License for the specific language governing permissions and
|
||
|
* limitations under the License.
|
||
|
*/
|
||
|
|
||
|
import (
|
||
|
"fmt"
|
||
|
"reflect"
|
||
|
"strings"
|
||
|
"unsafe"
|
||
|
)
|
||
|
|
||
|
/*
|
||
|
#include <stdlib.h>
|
||
|
#include "select_rdkafka.h"
|
||
|
*/
|
||
|
import "C"
|
||
|
|
||
|
// ConfigValue supports the following types:
|
||
|
// bool, int, string, any type with the standard String() interface
|
||
|
type ConfigValue interface{}
|
||
|
|
||
|
// ConfigMap is a map containing standard librdkafka configuration properties as documented in:
|
||
|
// https://github.com/edenhill/librdkafka/tree/master/CONFIGURATION.md
|
||
|
//
|
||
|
// The special property "default.topic.config" (optional) is a ConfigMap
|
||
|
// containing default topic configuration properties.
|
||
|
//
|
||
|
// The use of "default.topic.config" is deprecated,
|
||
|
// topic configuration properties shall be specified in the standard ConfigMap.
|
||
|
// For backwards compatibility, "default.topic.config" (if supplied)
|
||
|
// takes precedence.
|
||
|
type ConfigMap map[string]ConfigValue
|
||
|
|
||
|
// SetKey sets configuration property key to value.
|
||
|
//
|
||
|
// For user convenience a key prefixed with {topic}. will be
|
||
|
// set on the "default.topic.config" sub-map, this use is deprecated.
|
||
|
func (m ConfigMap) SetKey(key string, value ConfigValue) error {
|
||
|
if strings.HasPrefix(key, "{topic}.") {
|
||
|
_, found := m["default.topic.config"]
|
||
|
if !found {
|
||
|
m["default.topic.config"] = ConfigMap{}
|
||
|
}
|
||
|
m["default.topic.config"].(ConfigMap)[strings.TrimPrefix(key, "{topic}.")] = value
|
||
|
} else {
|
||
|
m[key] = value
|
||
|
}
|
||
|
|
||
|
return nil
|
||
|
}
|
||
|
|
||
|
// Set implements flag.Set (command line argument parser) as a convenience
|
||
|
// for `-X key=value` config.
|
||
|
func (m ConfigMap) Set(kv string) error {
|
||
|
i := strings.Index(kv, "=")
|
||
|
if i == -1 {
|
||
|
return newErrorFromString(ErrInvalidArg, "Expected key=value")
|
||
|
}
|
||
|
|
||
|
k := kv[:i]
|
||
|
v := kv[i+1:]
|
||
|
|
||
|
return m.SetKey(k, v)
|
||
|
}
|
||
|
|
||
|
func value2string(v ConfigValue) (ret string, errstr string) {
|
||
|
|
||
|
switch x := v.(type) {
|
||
|
case bool:
|
||
|
if x {
|
||
|
ret = "true"
|
||
|
} else {
|
||
|
ret = "false"
|
||
|
}
|
||
|
case int:
|
||
|
ret = fmt.Sprintf("%d", x)
|
||
|
case string:
|
||
|
ret = x
|
||
|
case fmt.Stringer:
|
||
|
ret = x.String()
|
||
|
default:
|
||
|
return "", fmt.Sprintf("Invalid value type %T", v)
|
||
|
}
|
||
|
|
||
|
return ret, ""
|
||
|
}
|
||
|
|
||
|
// rdkAnyconf abstracts rd_kafka_conf_t and rd_kafka_topic_conf_t
|
||
|
// into a common interface.
|
||
|
type rdkAnyconf interface {
|
||
|
set(cKey *C.char, cVal *C.char, cErrstr *C.char, errstrSize int) C.rd_kafka_conf_res_t
|
||
|
}
|
||
|
|
||
|
func anyconfSet(anyconf rdkAnyconf, key string, val ConfigValue) (err error) {
|
||
|
value, errstr := value2string(val)
|
||
|
if errstr != "" {
|
||
|
return newErrorFromString(ErrInvalidArg, fmt.Sprintf("%s for key %s (expected string,bool,int,ConfigMap)", errstr, key))
|
||
|
}
|
||
|
cKey := C.CString(key)
|
||
|
defer C.free(unsafe.Pointer(cKey))
|
||
|
cVal := C.CString(value)
|
||
|
defer C.free(unsafe.Pointer(cVal))
|
||
|
cErrstr := (*C.char)(C.malloc(C.size_t(128)))
|
||
|
defer C.free(unsafe.Pointer(cErrstr))
|
||
|
|
||
|
if anyconf.set(cKey, cVal, cErrstr, 128) != C.RD_KAFKA_CONF_OK {
|
||
|
return newErrorFromCString(C.RD_KAFKA_RESP_ERR__INVALID_ARG, cErrstr)
|
||
|
}
|
||
|
|
||
|
return nil
|
||
|
}
|
||
|
|
||
|
// we need these typedefs to workaround a crash in golint
|
||
|
// when parsing the set() methods below
|
||
|
type rdkConf C.rd_kafka_conf_t
|
||
|
type rdkTopicConf C.rd_kafka_topic_conf_t
|
||
|
|
||
|
func (cConf *rdkConf) set(cKey *C.char, cVal *C.char, cErrstr *C.char, errstrSize int) C.rd_kafka_conf_res_t {
|
||
|
return C.rd_kafka_conf_set((*C.rd_kafka_conf_t)(cConf), cKey, cVal, cErrstr, C.size_t(errstrSize))
|
||
|
}
|
||
|
|
||
|
func (ctopicConf *rdkTopicConf) set(cKey *C.char, cVal *C.char, cErrstr *C.char, errstrSize int) C.rd_kafka_conf_res_t {
|
||
|
return C.rd_kafka_topic_conf_set((*C.rd_kafka_topic_conf_t)(ctopicConf), cKey, cVal, cErrstr, C.size_t(errstrSize))
|
||
|
}
|
||
|
|
||
|
func configConvertAnyconf(m ConfigMap, anyconf rdkAnyconf) (err error) {
|
||
|
// set plugins first, any plugin-specific configuration depends on
|
||
|
// the plugin to have already been set
|
||
|
pluginPaths, ok := m["plugin.library.paths"]
|
||
|
if ok {
|
||
|
err = anyconfSet(anyconf, "plugin.library.paths", pluginPaths)
|
||
|
if err != nil {
|
||
|
return err
|
||
|
}
|
||
|
}
|
||
|
for k, v := range m {
|
||
|
if k == "plugin.library.paths" {
|
||
|
continue
|
||
|
}
|
||
|
switch v.(type) {
|
||
|
case ConfigMap:
|
||
|
/* Special sub-ConfigMap, only used for default.topic.config */
|
||
|
|
||
|
if k != "default.topic.config" {
|
||
|
return newErrorFromString(ErrInvalidArg, fmt.Sprintf("Invalid type for key %s", k))
|
||
|
}
|
||
|
|
||
|
var cTopicConf = C.rd_kafka_topic_conf_new()
|
||
|
|
||
|
err = configConvertAnyconf(v.(ConfigMap),
|
||
|
(*rdkTopicConf)(cTopicConf))
|
||
|
if err != nil {
|
||
|
C.rd_kafka_topic_conf_destroy(cTopicConf)
|
||
|
return err
|
||
|
}
|
||
|
|
||
|
C.rd_kafka_conf_set_default_topic_conf(
|
||
|
(*C.rd_kafka_conf_t)(anyconf.(*rdkConf)),
|
||
|
(*C.rd_kafka_topic_conf_t)((*rdkTopicConf)(cTopicConf)))
|
||
|
|
||
|
default:
|
||
|
err = anyconfSet(anyconf, k, v)
|
||
|
if err != nil {
|
||
|
return err
|
||
|
}
|
||
|
}
|
||
|
}
|
||
|
|
||
|
return nil
|
||
|
}
|
||
|
|
||
|
// convert ConfigMap to C rd_kafka_conf_t *
|
||
|
func (m ConfigMap) convert() (cConf *C.rd_kafka_conf_t, err error) {
|
||
|
cConf = C.rd_kafka_conf_new()
|
||
|
|
||
|
// Set the client.software.name and .version (use librdkafka version).
|
||
|
_, librdkafkaVersion := LibraryVersion()
|
||
|
anyconfSet((*rdkConf)(cConf), "client.software.name", "confluent-kafka-go")
|
||
|
anyconfSet((*rdkConf)(cConf), "client.software.version", librdkafkaVersion)
|
||
|
|
||
|
err = configConvertAnyconf(m, (*rdkConf)(cConf))
|
||
|
if err != nil {
|
||
|
C.rd_kafka_conf_destroy(cConf)
|
||
|
return nil, err
|
||
|
}
|
||
|
return cConf, nil
|
||
|
}
|
||
|
|
||
|
// get finds key in the configmap and returns its value.
|
||
|
// If the key is not found defval is returned.
|
||
|
// If the key is found but the type is mismatched an error is returned.
|
||
|
func (m ConfigMap) get(key string, defval ConfigValue) (ConfigValue, error) {
|
||
|
if strings.HasPrefix(key, "{topic}.") {
|
||
|
defconfCv, found := m["default.topic.config"]
|
||
|
if !found {
|
||
|
return defval, nil
|
||
|
}
|
||
|
return defconfCv.(ConfigMap).get(strings.TrimPrefix(key, "{topic}."), defval)
|
||
|
}
|
||
|
|
||
|
v, ok := m[key]
|
||
|
if !ok {
|
||
|
return defval, nil
|
||
|
}
|
||
|
|
||
|
if defval != nil && reflect.TypeOf(defval) != reflect.TypeOf(v) {
|
||
|
return nil, newErrorFromString(ErrInvalidArg, fmt.Sprintf("%s expects type %T, not %T", key, defval, v))
|
||
|
}
|
||
|
|
||
|
return v, nil
|
||
|
}
|
||
|
|
||
|
// extract performs a get() and if found deletes the key.
|
||
|
func (m ConfigMap) extract(key string, defval ConfigValue) (ConfigValue, error) {
|
||
|
|
||
|
v, err := m.get(key, defval)
|
||
|
if err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
delete(m, key)
|
||
|
|
||
|
return v, nil
|
||
|
}
|
||
|
|
||
|
// extractLogConfig extracts generic go.logs.* configuration properties.
|
||
|
func (m ConfigMap) extractLogConfig() (logsChanEnable bool, logsChan chan LogEvent, err error) {
|
||
|
v, err := m.extract("go.logs.channel.enable", false)
|
||
|
if err != nil {
|
||
|
return
|
||
|
}
|
||
|
|
||
|
logsChanEnable = v.(bool)
|
||
|
|
||
|
v, err = m.extract("go.logs.channel", nil)
|
||
|
if err != nil {
|
||
|
return
|
||
|
}
|
||
|
|
||
|
if v != nil {
|
||
|
logsChan = v.(chan LogEvent)
|
||
|
}
|
||
|
|
||
|
if logsChanEnable {
|
||
|
// Tell librdkafka to forward logs to the log queue
|
||
|
m.Set("log.queue=true")
|
||
|
}
|
||
|
|
||
|
return
|
||
|
}
|
||
|
|
||
|
func (m ConfigMap) clone() ConfigMap {
|
||
|
m2 := make(ConfigMap)
|
||
|
for k, v := range m {
|
||
|
m2[k] = v
|
||
|
}
|
||
|
return m2
|
||
|
}
|
||
|
|
||
|
// Get finds the given key in the ConfigMap and returns its value.
|
||
|
// If the key is not found `defval` is returned.
|
||
|
// If the key is found but the type does not match that of `defval` (unless nil)
|
||
|
// an ErrInvalidArg error is returned.
|
||
|
func (m ConfigMap) Get(key string, defval ConfigValue) (ConfigValue, error) {
|
||
|
return m.get(key, defval)
|
||
|
}
|