You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

280 lines
7.5 KiB

package kafka
/**
* Copyright 2016 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import (
"fmt"
"reflect"
"strings"
"unsafe"
)
/*
#include <stdlib.h>
#include "select_rdkafka.h"
*/
import "C"
// ConfigValue supports the following types:
// bool, int, string, any type with the standard String() interface
type ConfigValue interface{}
// ConfigMap is a map containing standard librdkafka configuration properties as documented in:
// https://github.com/edenhill/librdkafka/tree/master/CONFIGURATION.md
//
// The special property "default.topic.config" (optional) is a ConfigMap
// containing default topic configuration properties.
//
// The use of "default.topic.config" is deprecated,
// topic configuration properties shall be specified in the standard ConfigMap.
// For backwards compatibility, "default.topic.config" (if supplied)
// takes precedence.
type ConfigMap map[string]ConfigValue
// SetKey sets configuration property key to value.
//
// For user convenience a key prefixed with {topic}. will be
// set on the "default.topic.config" sub-map, this use is deprecated.
func (m ConfigMap) SetKey(key string, value ConfigValue) error {
if strings.HasPrefix(key, "{topic}.") {
_, found := m["default.topic.config"]
if !found {
m["default.topic.config"] = ConfigMap{}
}
m["default.topic.config"].(ConfigMap)[strings.TrimPrefix(key, "{topic}.")] = value
} else {
m[key] = value
}
return nil
}
// Set implements flag.Set (command line argument parser) as a convenience
// for `-X key=value` config.
func (m ConfigMap) Set(kv string) error {
i := strings.Index(kv, "=")
if i == -1 {
return newErrorFromString(ErrInvalidArg, "Expected key=value")
}
k := kv[:i]
v := kv[i+1:]
return m.SetKey(k, v)
}
func value2string(v ConfigValue) (ret string, errstr string) {
switch x := v.(type) {
case bool:
if x {
ret = "true"
} else {
ret = "false"
}
case int:
ret = fmt.Sprintf("%d", x)
case string:
ret = x
case fmt.Stringer:
ret = x.String()
default:
return "", fmt.Sprintf("Invalid value type %T", v)
}
return ret, ""
}
// rdkAnyconf abstracts rd_kafka_conf_t and rd_kafka_topic_conf_t
// into a common interface.
type rdkAnyconf interface {
set(cKey *C.char, cVal *C.char, cErrstr *C.char, errstrSize int) C.rd_kafka_conf_res_t
}
func anyconfSet(anyconf rdkAnyconf, key string, val ConfigValue) (err error) {
value, errstr := value2string(val)
if errstr != "" {
return newErrorFromString(ErrInvalidArg, fmt.Sprintf("%s for key %s (expected string,bool,int,ConfigMap)", errstr, key))
}
cKey := C.CString(key)
defer C.free(unsafe.Pointer(cKey))
cVal := C.CString(value)
defer C.free(unsafe.Pointer(cVal))
cErrstr := (*C.char)(C.malloc(C.size_t(128)))
defer C.free(unsafe.Pointer(cErrstr))
if anyconf.set(cKey, cVal, cErrstr, 128) != C.RD_KAFKA_CONF_OK {
return newErrorFromCString(C.RD_KAFKA_RESP_ERR__INVALID_ARG, cErrstr)
}
return nil
}
// we need these typedefs to workaround a crash in golint
// when parsing the set() methods below
type rdkConf C.rd_kafka_conf_t
type rdkTopicConf C.rd_kafka_topic_conf_t
func (cConf *rdkConf) set(cKey *C.char, cVal *C.char, cErrstr *C.char, errstrSize int) C.rd_kafka_conf_res_t {
return C.rd_kafka_conf_set((*C.rd_kafka_conf_t)(cConf), cKey, cVal, cErrstr, C.size_t(errstrSize))
}
func (ctopicConf *rdkTopicConf) set(cKey *C.char, cVal *C.char, cErrstr *C.char, errstrSize int) C.rd_kafka_conf_res_t {
return C.rd_kafka_topic_conf_set((*C.rd_kafka_topic_conf_t)(ctopicConf), cKey, cVal, cErrstr, C.size_t(errstrSize))
}
func configConvertAnyconf(m ConfigMap, anyconf rdkAnyconf) (err error) {
// set plugins first, any plugin-specific configuration depends on
// the plugin to have already been set
pluginPaths, ok := m["plugin.library.paths"]
if ok {
err = anyconfSet(anyconf, "plugin.library.paths", pluginPaths)
if err != nil {
return err
}
}
for k, v := range m {
if k == "plugin.library.paths" {
continue
}
switch v.(type) {
case ConfigMap:
/* Special sub-ConfigMap, only used for default.topic.config */
if k != "default.topic.config" {
return newErrorFromString(ErrInvalidArg, fmt.Sprintf("Invalid type for key %s", k))
}
var cTopicConf = C.rd_kafka_topic_conf_new()
err = configConvertAnyconf(v.(ConfigMap),
(*rdkTopicConf)(cTopicConf))
if err != nil {
C.rd_kafka_topic_conf_destroy(cTopicConf)
return err
}
C.rd_kafka_conf_set_default_topic_conf(
(*C.rd_kafka_conf_t)(anyconf.(*rdkConf)),
(*C.rd_kafka_topic_conf_t)((*rdkTopicConf)(cTopicConf)))
default:
err = anyconfSet(anyconf, k, v)
if err != nil {
return err
}
}
}
return nil
}
// convert ConfigMap to C rd_kafka_conf_t *
func (m ConfigMap) convert() (cConf *C.rd_kafka_conf_t, err error) {
cConf = C.rd_kafka_conf_new()
// Set the client.software.name and .version (use librdkafka version).
_, librdkafkaVersion := LibraryVersion()
anyconfSet((*rdkConf)(cConf), "client.software.name", "confluent-kafka-go")
anyconfSet((*rdkConf)(cConf), "client.software.version", librdkafkaVersion)
err = configConvertAnyconf(m, (*rdkConf)(cConf))
if err != nil {
C.rd_kafka_conf_destroy(cConf)
return nil, err
}
return cConf, nil
}
// get finds key in the configmap and returns its value.
// If the key is not found defval is returned.
// If the key is found but the type is mismatched an error is returned.
func (m ConfigMap) get(key string, defval ConfigValue) (ConfigValue, error) {
if strings.HasPrefix(key, "{topic}.") {
defconfCv, found := m["default.topic.config"]
if !found {
return defval, nil
}
return defconfCv.(ConfigMap).get(strings.TrimPrefix(key, "{topic}."), defval)
}
v, ok := m[key]
if !ok {
return defval, nil
}
if defval != nil && reflect.TypeOf(defval) != reflect.TypeOf(v) {
return nil, newErrorFromString(ErrInvalidArg, fmt.Sprintf("%s expects type %T, not %T", key, defval, v))
}
return v, nil
}
// extract performs a get() and if found deletes the key.
func (m ConfigMap) extract(key string, defval ConfigValue) (ConfigValue, error) {
v, err := m.get(key, defval)
if err != nil {
return nil, err
}
delete(m, key)
return v, nil
}
// extractLogConfig extracts generic go.logs.* configuration properties.
func (m ConfigMap) extractLogConfig() (logsChanEnable bool, logsChan chan LogEvent, err error) {
v, err := m.extract("go.logs.channel.enable", false)
if err != nil {
return
}
logsChanEnable = v.(bool)
v, err = m.extract("go.logs.channel", nil)
if err != nil {
return
}
if v != nil {
logsChan = v.(chan LogEvent)
}
if logsChanEnable {
// Tell librdkafka to forward logs to the log queue
m.Set("log.queue=true")
}
return
}
func (m ConfigMap) clone() ConfigMap {
m2 := make(ConfigMap)
for k, v := range m {
m2[k] = v
}
return m2
}
// Get finds the given key in the ConfigMap and returns its value.
// If the key is not found `defval` is returned.
// If the key is found but the type does not match that of `defval` (unless nil)
// an ErrInvalidArg error is returned.
func (m ConfigMap) Get(key string, defval ConfigValue) (ConfigValue, error) {
return m.get(key, defval)
}