Browse Source

Init

master
Andrey Kovalev 3 years ago
commit
d838fbf2b4
Signed by: Russia9
GPG Key ID: EC3AA65373CA51DC
  1. 37
      .drone.yml
  2. 63
      .gitignore
  3. 13
      Dockerfile
  4. 674
      LICENSE
  5. 2
      README.md
  6. 170
      cmd/main/main.go
  7. 26
      go.mod
  8. 96
      go.sum
  9. 10
      internal/utils/common.go
  10. 22
      pkg/domain/user.go
  11. 13
      pkg/message/deal.go
  12. 17
      pkg/message/duel.go
  13. 20
      pkg/message/lot.go
  14. 10
      pkg/message/offer.go
  15. 23
      pkg/message/store.go
  16. 52
      pkg/storage/mongodb/user.go
  17. 202
      vendor/github.com/confluentinc/confluent-kafka-go/LICENSE
  18. 2
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/.gitignore
  19. 58
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/00version.go
  20. 145
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/README.md
  21. 1025
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/adminapi.go
  22. 264
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/adminoptions.go
  23. 4737
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/api.html
  24. 13
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/build_darwin.go
  25. 9
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/build_dynamic.go
  26. 13
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/build_glibc_linux.go
  27. 13
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/build_musl_linux.go
  28. 13
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/build_windows.go
  29. 280
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/config.go
  30. 923
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/consumer.go
  31. 31
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/context.go
  32. 154
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/error.go
  33. 112
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/error_gen.go
  34. 316
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/event.go
  35. 337
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/generated_errors.go
  36. 48
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/glue_rdkafka.h
  37. 379
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/handle.go
  38. 67
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/header.go
  39. 375
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/kafka.go
  40. 3
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/.gitignore
  41. 366
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/LICENSES.txt
  42. 24
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/README.md
  43. 113
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/bundle-import.sh
  44. 110
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/import.sh
  45. 21
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/librdkafka.go
  46. BIN
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/librdkafka_darwin.a
  47. BIN
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/librdkafka_glibc_linux.a
  48. BIN
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/librdkafka_musl_linux.a
  49. BIN
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/librdkafka_windows.a
  50. 7781
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/rdkafka.h
  51. 89
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/log.go
  52. 223
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/message.go
  53. 180
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/metadata.go
  54. 35
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/misc.go
  55. 145
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/offset.go
  56. 918
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/producer.go
  57. 29
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/select_rdkafka.h
  58. 8
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/testconf-example.json
  59. 248
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/testhelpers.go
  60. 55
      vendor/github.com/confluentinc/confluent-kafka-go/kafka/time.go
  61. 21
      vendor/github.com/go-stack/stack/LICENSE.md
  62. 38
      vendor/github.com/go-stack/stack/README.md
  63. 400
      vendor/github.com/go-stack/stack/stack.go
  64. 16
      vendor/github.com/golang/snappy/.gitignore
  65. 18
      vendor/github.com/golang/snappy/AUTHORS
  66. 41
      vendor/github.com/golang/snappy/CONTRIBUTORS
  67. 27
      vendor/github.com/golang/snappy/LICENSE
  68. 107
      vendor/github.com/golang/snappy/README
  69. 264
      vendor/github.com/golang/snappy/decode.go
  70. 490
      vendor/github.com/golang/snappy/decode_amd64.s
  71. 494
      vendor/github.com/golang/snappy/decode_arm64.s
  72. 15
      vendor/github.com/golang/snappy/decode_asm.go
  73. 115
      vendor/github.com/golang/snappy/decode_other.go
  74. 289
      vendor/github.com/golang/snappy/encode.go
  75. 730
      vendor/github.com/golang/snappy/encode_amd64.s
  76. 722
      vendor/github.com/golang/snappy/encode_arm64.s
  77. 30
      vendor/github.com/golang/snappy/encode_asm.go
  78. 238
      vendor/github.com/golang/snappy/encode_other.go
  79. 98
      vendor/github.com/golang/snappy/snappy.go
  80. 9
      vendor/github.com/google/uuid/.travis.yml
  81. 10
      vendor/github.com/google/uuid/CONTRIBUTING.md
  82. 9
      vendor/github.com/google/uuid/CONTRIBUTORS
  83. 27
      vendor/github.com/google/uuid/LICENSE
  84. 19
      vendor/github.com/google/uuid/README.md
  85. 80
      vendor/github.com/google/uuid/dce.go
  86. 12
      vendor/github.com/google/uuid/doc.go
  87. 53
      vendor/github.com/google/uuid/hash.go
  88. 38
      vendor/github.com/google/uuid/marshal.go
  89. 90
      vendor/github.com/google/uuid/node.go
  90. 12
      vendor/github.com/google/uuid/node_js.go
  91. 33
      vendor/github.com/google/uuid/node_net.go
  92. 118
      vendor/github.com/google/uuid/null.go
  93. 59
      vendor/github.com/google/uuid/sql.go
  94. 123
      vendor/github.com/google/uuid/time.go
  95. 43
      vendor/github.com/google/uuid/util.go
  96. 294
      vendor/github.com/google/uuid/uuid.go
  97. 44
      vendor/github.com/google/uuid/version1.go
  98. 76
      vendor/github.com/google/uuid/version4.go
  99. 2
      vendor/github.com/klauspost/compress/.gitattributes
  100. 25
      vendor/github.com/klauspost/compress/.gitignore
  101. Some files were not shown because too many files have changed in this diff Show More

37
.drone.yml

@ -0,0 +1,37 @@
kind: pipeline
type: docker
name: build
steps:
- name: build
image: plugins/docker
settings:
registry: cr.selcloud.ru
username:
from_secret: docker_username
password:
from_secret: docker_password
repo: cr.selcloud.ru/russia9/${DRONE_REPO_NAME}/${DRONE_COMMIT_BRANCH}
tags:
- latest
- ${DRONE_COMMIT_SHA}
cache_from:
- cr.selcloud.ru/russia9/${DRONE_REPO_NAME}/${DRONE_COMMIT_BRANCH}:latest
- name: deploy
image: appleboy/drone-ssh
settings:
host:
from_secret: ssh_address
username:
from_secret: ssh_username
key:
from_secret: ssh_key
port: 22
script_stop: true
script:
- docker-compose -f /srv/chatwars-spy/docker-compose.yml pull
- docker-compose -f /srv/chatwars-spy/docker-compose.yml up -d
when:
branch:
- master

63
.gitignore vendored

@ -0,0 +1,63 @@
# Created by .ignore support plugin (hsz.mobi)
### Go template
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Dependency directories (remove the comment below to include it)
# vendor/
### Vim template
# Swap
[._]*.s[a-v][a-z]
!*.svg # comment out if you don't need vector files
[._]*.sw[a-p]
[._]s[a-rt-v][a-z]
[._]ss[a-gi-z]
[._]sw[a-p]
# Session
Session.vim
Sessionx.vim
# Temporary
.netrwhist
*~
# Auto-generated tag files
tags
# Persistent undo
[._]*.un~
### JetBrains template
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# User-specific stuff
.idea
# CMake
cmake-build-*/
# File-based project format
*.iws
# IntelliJ
out/
# JIRA plugin
atlassian-ide-plugin.xml
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties

13
Dockerfile

@ -0,0 +1,13 @@
FROM golang:1.18
# Set app workdir
WORKDIR /go/src/app
# Copy application sources
COPY . .
# Build app
RUN go build -o app gitea.russia9.dev/Russia9/chatwars-spy/cmd/main
# Run app
CMD ["./app"]

674
LICENSE

@ -0,0 +1,674 @@
GNU GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The GNU General Public License is a free, copyleft license for
software and other kinds of works.
The licenses for most software and other practical works are designed
to take away your freedom to share and change the works. By contrast,
the GNU General Public License is intended to guarantee your freedom to
share and change all versions of a program--to make sure it remains free
software for all its users. We, the Free Software Foundation, use the
GNU General Public License for most of our software; it applies also to
any other work released this way by its authors. You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
them if you wish), that you receive source code or can get it if you
want it, that you can change the software or use pieces of it in new
free programs, and that you know you can do these things.
To protect your rights, we need to prevent others from denying you
these rights or asking you to surrender the rights. Therefore, you have
certain responsibilities if you distribute copies of the software, or if
you modify it: responsibilities to respect the freedom of others.
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must pass on to the recipients the same
freedoms that you received. You must make sure that they, too, receive
or can get the source code. And you must show them these terms so they
know their rights.
Developers that use the GNU GPL protect your rights with two steps:
(1) assert copyright on the software, and (2) offer you this License
giving you legal permission to copy, distribute and/or modify it.
For the developers' and authors' protection, the GPL clearly explains
that there is no warranty for this free software. For both users' and
authors' sake, the GPL requires that modified versions be marked as
changed, so that their problems will not be attributed erroneously to
authors of previous versions.
Some devices are designed to deny users access to install or run
modified versions of the software inside them, although the manufacturer
can do so. This is fundamentally incompatible with the aim of
protecting users' freedom to change the software. The systematic
pattern of such abuse occurs in the area of products for individuals to
use, which is precisely where it is most unacceptable. Therefore, we
have designed this version of the GPL to prohibit the practice for those
products. If such problems arise substantially in other domains, we
stand ready to extend this provision to those domains in future versions
of the GPL, as needed to protect the freedom of users.
Finally, every program is threatened constantly by software patents.
States should not allow patents to restrict development and use of
software on general-purpose computers, but in those that do, we wish to
avoid the special danger that patents applied to a free program could
make it effectively proprietary. To prevent this, the GPL assures that
patents cannot be used to render the program non-free.
The precise terms and conditions for copying, distribution and
modification follow.
TERMS AND CONDITIONS
0. Definitions.
"This License" refers to version 3 of the GNU General Public License.
"Copyright" also means copyright-like laws that apply to other kinds of
works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this
License. Each licensee is addressed as "you". "Licensees" and
"recipients" may be individuals or organizations.
To "modify" a work means to copy from or adapt all or part of the work
in a fashion requiring copyright permission, other than the making of an
exact copy. The resulting work is called a "modified version" of the
earlier work or a work "based on" the earlier work.
A "covered work" means either the unmodified Program or a work based
on the Program.
To "propagate" a work means to do anything with it that, without
permission, would make you directly or secondarily liable for
infringement under applicable copyright law, except executing it on a
computer or modifying a private copy. Propagation includes copying,
distribution (with or without modification), making available to the
public, and in some countries other activities as well.
To "convey" a work means any kind of propagation that enables other
parties to make or receive copies. Mere interaction with a user through
a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays "Appropriate Legal Notices"
to the extent that it includes a convenient and prominently visible
feature that (1) displays an appropriate copyright notice, and (2)
tells the user that there is no warranty for the work (except to the
extent that warranties are provided), that licensees may convey the
work under this License, and how to view a copy of this License. If
the interface presents a list of user commands or options, such as a
menu, a prominent item in the list meets this criterion.
1. Source Code.
The "source code" for a work means the preferred form of the work
for making modifications to it. "Object code" means any non-source
form of a work.
A "Standard Interface" means an interface that either is an official
standard defined by a recognized standards body, or, in the case of
interfaces specified for a particular programming language, one that
is widely used among developers working in that language.
The "System Libraries" of an executable work include anything, other
than the work as a whole, that (a) is included in the normal form of
packaging a Major Component, but which is not part of that Major
Component, and (b) serves only to enable use of the work with that
Major Component, or to implement a Standard Interface for which an
implementation is available to the public in source code form. A
"Major Component", in this context, means a major essential component
(kernel, window system, and so on) of the specific operating system
(if any) on which the executable work runs, or a compiler used to
produce the work, or an object code interpreter used to run it.
The "Corresponding Source" for a work in object code form means all
the source code needed to generate, install, and (for an executable
work) run the object code and to modify the work, including scripts to
control those activities. However, it does not include the work's
System Libraries, or general-purpose tools or generally available free
programs which are used unmodified in performing those activities but
which are not part of the work. For example, Corresponding Source
includes interface definition files associated with source files for
the work, and the source code for shared libraries and dynamically
linked subprograms that the work is specifically designed to require,
such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users
can regenerate automatically from other parts of the Corresponding
Source.
The Corresponding Source for a work in source code form is that
same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of
copyright on the Program, and are irrevocable provided the stated
conditions are met. This License explicitly affirms your unlimited
permission to run the unmodified Program. The output from running a
covered work is covered by this License only if the output, given its
content, constitutes a covered work. This License acknowledges your
rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not
convey, without conditions so long as your license otherwise remains
in force. You may convey covered works to others for the sole purpose
of having them make modifications exclusively for you, or provide you
with facilities for running those works, provided that you comply with
the terms of this License in conveying all material for which you do
not control copyright. Those thus making or running the covered works
for you must do so exclusively on your behalf, under your direction
and control, on terms that prohibit them from making any copies of
your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under
the conditions stated below. Sublicensing is not allowed; section 10
makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological
measure under any applicable law fulfilling obligations under article
11 of the WIPO copyright treaty adopted on 20 December 1996, or
similar laws prohibiting or restricting circumvention of such
measures.
When you convey a covered work, you waive any legal power to forbid
circumvention of technological measures to the extent such circumvention
is effected by exercising rights under this License with respect to
the covered work, and you disclaim any intention to limit operation or
modification of the work as a means of enforcing, against the work's
users, your or third parties' legal rights to forbid circumvention of
technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you
receive it, in any medium, provided that you conspicuously and
appropriately publish on each copy an appropriate copyright notice;
keep intact all notices stating that this License and any
non-permissive terms added in accord with section 7 apply to the code;
keep intact all notices of the absence of any warranty; and give all
recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey,
and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to
produce it from the Program, in the form of source code under the
terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified
it, and giving a relevant date.
b) The work must carry prominent notices stating that it is
released under this License and any conditions added under section
7. This requirement modifies the requirement in section 4 to
"keep intact all notices".
c) You must license the entire work, as a whole, under this
License to anyone who comes into possession of a copy. This
License will therefore apply, along with any applicable section 7
additional terms, to the whole of the work, and all its parts,
regardless of how they are packaged. This License gives no
permission to license the work in any other way, but it does not
invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display
Appropriate Legal Notices; however, if the Program has interactive
interfaces that do not display Appropriate Legal Notices, your
work need not make them do so.
A compilation of a covered work with other separate and independent
works, which are not by their nature extensions of the covered work,
and which are not combined with it such as to form a larger program,
in or on a volume of a storage or distribution medium, is called an
"aggregate" if the compilation and its resulting copyright are not
used to limit the access or legal rights of the compilation's users
beyond what the individual works permit. Inclusion of a covered work
in an aggregate does not cause this License to apply to the other
parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms
of sections 4 and 5, provided that you also convey the
machine-readable Corresponding Source under the terms of this License,
in one of these ways:
a) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by the
Corresponding Source fixed on a durable physical medium
customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by a
written offer, valid for at least three years and valid for as
long as you offer spare parts or customer support for that product
model, to give anyone who possesses the object code either (1) a
copy of the Corresponding Source for all the software in the
product that is covered by this License, on a durable physical
medium customarily used for software interchange, for a price no
more than your reasonable cost of physically performing this
conveying of source, or (2) access to copy the
Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the
written offer to provide the Corresponding Source. This
alternative is allowed only occasionally and noncommercially, and
only if you received the object code with such an offer, in accord
with subsection 6b.
d) Convey the object code by offering access from a designated
place (gratis or for a charge), and offer equivalent access to the
Corresponding Source in the same way through the same place at no
further charge. You need not require recipients to copy the
Corresponding Source along with the object code. If the place to
copy the object code is a network server, the Corresponding Source
may be on a different server (operated by you or a third party)
that supports equivalent copying facilities, provided you maintain
clear directions next to the object code saying where to find the
Corresponding Source. Regardless of what server hosts the
Corresponding Source, you remain obligated to ensure that it is
available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided
you inform other peers where the object code and Corresponding
Source of the work are being offered to the general public at no
charge under subsection 6d.
A separable portion of the object code, whose source code is excluded
from the Corresponding Source as a System Library, need not be
included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any
tangible personal property which is normally used for personal, family,
or household purposes, or (2) anything designed or sold for incorporation
into a dwelling. In determining whether a product is a consumer product,
doubtful cases shall be resolved in favor of coverage. For a particular
product received by a particular user, "normally used" refers to a
typical or common use of that class of product, regardless of the status
of the particular user or of the way in which the particular user
actually uses, or expects or is expected to use, the product. A product
is a consumer product regardless of whether the product has substantial
commercial, industrial or non-consumer uses, unless such uses represent
the only significant mode of use of the product.
"Installation Information" for a User Product means any methods,
procedures, authorization keys, or other information required to install
and execute modified versions of a covered work in that User Product from
a modified version of its Corresponding Source. The information must
suffice to ensure that the continued functioning of the modified object
code is in no case prevented or interfered with solely because
modification has been made.
If you convey an object code work under this section in, or with, or
specifically for use in, a User Product, and the conveying occurs as
part of a transaction in which the right of possession and use of the
User Product is transferred to the recipient in perpetuity or for a
fixed term (regardless of how the transaction is characterized), the
Corresponding Source conveyed under this section must be accompanied
by the Installation Information. But this requirement does not apply
if neither you nor any third party retains the ability to install
modified object code on the User Product (for example, the work has
been installed in ROM).
The requirement to provide Installation Information does not include a
requirement to continue to provide support service, warranty, or updates
for a work that has been modified or installed by the recipient, or for
the User Product in which it has been modified or installed. Access to a
network may be denied when the modification itself materially and
adversely affects the operation of the network or violates the rules and
protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided,
in accord with this section must be in a format that is publicly
documented (and with an implementation available to the public in
source code form), and must require no special password or key for
unpacking, reading or copying.
7. Additional Terms.
"Additional permissions" are terms that supplement the terms of this
License by making exceptions from one or more of its conditions.
Additional permissions that are applicable to the entire Program shall
be treated as though they were included in this License, to the extent
that they are valid under applicable law. If additional permissions
apply only to part of the Program, that part may be used separately
under those permissions, but the entire Program remains governed by
this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option
remove any additional permissions from that copy, or from any part of
it. (Additional permissions may be written to require their own
removal in certain cases when you modify the work.) You may place
additional permissions on material, added by you to a covered work,
for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you
add to a covered work, you may (if authorized by the copyright holders of
that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the
terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or
author attributions in that material or in the Appropriate Legal
Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or
requiring that modified versions of such material be marked in
reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or
authors of the material; or
e) Declining to grant rights under trademark law for use of some
trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that
material by anyone who conveys the material (or modified versions of
it) with contractual assumptions of liability to the recipient, for
any liability that these contractual assumptions directly impose on
those licensors and authors.
All other non-permissive additional terms are considered "further
restrictions" within the meaning of section 10. If the Program as you
received it, or any part of it, contains a notice stating that it is
governed by this License along with a term that is a further
restriction, you may remove that term. If a license document contains
a further restriction but permits relicensing or conveying under this
License, you may add to a covered work material governed by the terms
of that license document, provided that the further restriction does
not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you
must place, in the relevant source files, a statement of the
additional terms that apply to those files, or a notice indicating
where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the
form of a separately written license, or stated as exceptions;
the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly
provided under this License. Any attempt otherwise to propagate or
modify it is void, and will automatically terminate your rights under
this License (including any patent licenses granted under the third
paragraph of section 11).
However, if you cease all violation of this License, then your
license from a particular copyright holder is reinstated (a)
provisionally, unless and until the copyright holder explicitly and
finally terminates your license, and (b) permanently, if the copyright
holder fails to notify you of the violation by some reasonable means
prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is
reinstated permanently if the copyright holder notifies you of the
violation by some reasonable means, this is the first time you have
received notice of violation of this License (for any work) from that
copyright holder, and you cure the violation prior to 30 days after
your receipt of the notice.
Termination of your rights under this section does not terminate the
licenses of parties who have received copies or rights from you under
this License. If your rights have been terminated and not permanently
reinstated, you do not qualify to receive new licenses for the same
material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or
run a copy of the Program. Ancillary propagation of a covered work
occurring solely as a consequence of using peer-to-peer transmission
to receive a copy likewise does not require acceptance. However,
nothing other than this License grants you permission to propagate or
modify any covered work. These actions infringe copyright if you do
not accept this License. Therefore, by modifying or propagating a
covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically
receives a license from the original licensors, to run, modify and
propagate that work, subject to this License. You are not responsible
for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an
organization, or substantially all assets of one, or subdividing an
organization, or merging organizations. If propagation of a covered
work results from an entity transaction, each party to that
transaction who receives a copy of the work also receives whatever
licenses to the work the party's predecessor in interest had or could
give under the previous paragraph, plus a right to possession of the
Corresponding Source of the work from the predecessor in interest, if
the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the
rights granted or affirmed under this License. For example, you may
not impose a license fee, royalty, or other charge for exercise of
rights granted under this License, and you may not initiate litigation
(including a cross-claim or counterclaim in a lawsuit) alleging that
any patent claim is infringed by making, using, selling, offering for
sale, or importing the Program or any portion of it.
11. Patents.
A "contributor" is a copyright holder who authorizes use under this
License of the Program or a work on which the Program is based. The
work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims
owned or controlled by the contributor, whether already acquired or
hereafter acquired, that would be infringed by some manner, permitted
by this License, of making, using, or selling its contributor version,
but do not include claims that would be infringed only as a
consequence of further modification of the contributor version. For
purposes of this definition, "control" includes the right to grant
patent sublicenses in a manner consistent with the requirements of
this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free
patent license under the contributor's essential patent claims, to
make, use, sell, offer for sale, import and otherwise run, modify and
propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express
agreement or commitment, however denominated, not to enforce a patent
(such as an express permission to practice a patent or covenant not to
sue for patent infringement). To "grant" such a patent license to a
party means to make such an agreement or commitment not to enforce a
patent against the party.
If you convey a covered work, knowingly relying on a patent license,
and the Corresponding Source of the work is not available for anyone
to copy, free of charge and under the terms of this License, through a
publicly available network server or other readily accessible means,
then you must either (1) cause the Corresponding Source to be so
available, or (2) arrange to deprive yourself of the benefit of the
patent license for this particular work, or (3) arrange, in a manner
consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have
actual knowledge that, but for the patent license, your conveying the
covered work in a country, or your recipient's use of the covered work
in a country, would infringe one or more identifiable patents in that
country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or
arrangement, you convey, or propagate by procuring conveyance of, a
covered work, and grant a patent license to some of the parties
receiving the covered work authorizing them to use, propagate, modify
or convey a specific copy of the covered work, then the patent license
you grant is automatically extended to all recipients of the covered
work and works based on it.
A patent license is "discriminatory" if it does not include within
the scope of its coverage, prohibits the exercise of, or is
conditioned on the non-exercise of one or more of the rights that are
specifically granted under this License. You may not convey a covered
work if you are a party to an arrangement with a third party that is
in the business of distributing software, under which you make payment
to the third party based on the extent of your activity of conveying
the work, and under which the third party grants, to any of the
parties who would receive the covered work from you, a discriminatory
patent license (a) in connection with copies of the covered work
conveyed by you (or copies made from those copies), or (b) primarily
for and in connection with specific products or compilations that
contain the covered work, unless you entered into that arrangement,
or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting
any implied license or other defenses to infringement that may
otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot convey a
covered work so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you may
not convey it at all. For example, if you agree to terms that obligate you
to collect a royalty for further conveying from those to whom you convey
the Program, the only way you could satisfy both those terms and this
License would be to refrain entirely from conveying the Program.
13. Use with the GNU Affero General Public License.
Notwithstanding any other provision of this License, you have
permission to link or combine any covered work with a work licensed
under version 3 of the GNU Affero General Public License into a single
combined work, and to convey the resulting work. The terms of this
License will continue to apply to the part which is the covered work,
but the special requirements of the GNU Affero General Public License,
section 13, concerning interaction through a network will apply to the
combination as such.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of
the GNU General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the
Program specifies that a certain numbered version of the GNU General
Public License "or any later version" applies to it, you have the
option of following the terms and conditions either of that numbered
version or of any later version published by the Free Software
Foundation. If the Program does not specify a version number of the
GNU General Public License, you may choose any version ever published
by the Free Software Foundation.
If the Program specifies that a proxy can decide which future
versions of the GNU General Public License can be used, that proxy's
public statement of acceptance of a version permanently authorizes you
to choose that version for the Program.
Later license versions may give you additional or different
permissions. However, no additional obligations are imposed on any
author or copyright holder as a result of your choosing to follow a
later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided
above cannot be given local legal effect according to their terms,
reviewing courts shall apply local law that most closely approximates
an absolute waiver of all civil liability in connection with the
Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If the program does terminal interaction, make it output a short
notice like this when it starts in an interactive mode:
<program> Copyright (C) <year> <name of author>
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License. Of course, your program's commands
might be different; for a GUI interface, you would use an "about box".
You should also get your employer (if you work as a programmer) or school,
if any, to sign a "copyright disclaimer" for the program, if necessary.
For more information on this, and how to apply and follow the GNU GPL, see
<https://www.gnu.org/licenses/>.
The GNU General Public License does not permit incorporating your program
into proprietary programs. If your program is a subroutine library, you
may consider it more useful to permit linking proprietary applications with
the library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License. But first, please read
<https://www.gnu.org/licenses/why-not-lgpl.html>.

2
README.md

@ -0,0 +1,2 @@
# ChatWars Spy
Ищет пользователей ChatWars из информации о дуэлях, предложениях и сделках на бирже.

170
cmd/main/main.go

@ -0,0 +1,170 @@
package main
import (
"context"
"encoding/json"
"gitea.russia9.dev/Russia9/chatwars-spy/internal/utils"
"gitea.russia9.dev/Russia9/chatwars-spy/pkg/domain"
"gitea.russia9.dev/Russia9/chatwars-spy/pkg/message"
"gitea.russia9.dev/Russia9/chatwars-spy/pkg/storage/mongodb"
"github.com/confluentinc/confluent-kafka-go/kafka"
"github.com/google/uuid"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"os"
"strconv"
)
func main() {
// Log settings
zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
pretty, err := strconv.ParseBool(os.Getenv("LOG_PRETTY"))
if err != nil {
pretty = false
}
if pretty {
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
}
switch os.Getenv("LOG_LEVEL") {
case "DISABLED":
zerolog.SetGlobalLevel(zerolog.Disabled)
case "PANIC":
zerolog.SetGlobalLevel(zerolog.PanicLevel)
case "FATAL":
zerolog.SetGlobalLevel(zerolog.FatalLevel)
case "ERROR":
zerolog.SetGlobalLevel(zerolog.ErrorLevel)
case "WARN":
zerolog.SetGlobalLevel(zerolog.WarnLevel)
case "DEBUG":
zerolog.SetGlobalLevel(zerolog.DebugLevel)
case "TRACE":
zerolog.SetGlobalLevel(zerolog.TraceLevel)
default:
zerolog.SetGlobalLevel(zerolog.InfoLevel)
}
version := utils.GetEnv("CHATWARS_VERSION", "cw3")
// Kafka consumer init
consumer, err := kafka.NewConsumer(&kafka.ConfigMap{
"bootstrap.servers": utils.GetEnv("KAFKA_ADDRESS", "digest-api.chtwrs.com:9092"),
"group.id": version + "-" + uuid.New().String(),
"auto.offset.reset": "latest",
})
if err != nil {
log.Fatal().Err(err).Send()
}
// DB init
client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(os.Getenv("MONGO_URI")))
if err != nil {
log.Fatal().Err(err).Send()
}
defer client.Disconnect(context.Background())
err = client.Ping(context.Background(), nil)
if err != nil {
log.Fatal().Err(err).Send()
}
db := client.Database(utils.GetEnv("MONGO_DB", "chatwars"))
repo := mongodb.NewUserRepo(db)
// Kafka subscribe
err = consumer.SubscribeTopics([]string{version + "-offers", version + "-deals", version + "-duels"}, nil)
if err != nil {
log.Fatal().Err(err).Send()
}
log.Info().Msg("Started watching")
// Get messages
for {
// Read message
raw, err := consumer.ReadMessage(-1)
if err != nil {
log.Error().Str("source", "consumer").Err(err).Send()
continue
}
log.Trace().Bytes("message", raw.Value).Msg("received message")
switch *raw.TopicPartition.Topic {
case version + "-offers": // Offers
// Decode message
var msg message.Offer
err = json.Unmarshal(raw.Value, &msg)
if err != nil {
log.Warn().Err(err).Str("topic", *raw.TopicPartition.Topic).Send()
continue
}
// Store user
_ = repo.Store(context.Background(),
&domain.User{
ID: msg.SellerID,
Castle: msg.SellerCastle,
Name: msg.SellerName,
Source: "offers",
},
)
case version + "-duels": // Duels
// Decode message
var msg message.Duel
err = json.Unmarshal(raw.Value, &msg)
if err != nil {
log.Warn().Err(err).Str("topic", *raw.TopicPartition.Topic).Send()
continue
}
// Store users
_ = repo.Store(context.Background(),
&domain.User{
ID: msg.Winner.ID,
Castle: msg.Winner.Castle,
Guild: msg.Winner.Tag,
Name: msg.Winner.Name,
Level: msg.Winner.Level,
Source: "duels",
},
)
_ = repo.Store(context.Background(),
&domain.User{
ID: msg.Loser.ID,
Castle: msg.Loser.Castle,
Guild: msg.Loser.Tag,
Name: msg.Loser.Name,
Level: msg.Loser.Level,
Source: "duels",
},
)
case version + "-deals": // Deals
// Decode message
var msg message.Deal
err = json.Unmarshal(raw.Value, &msg)
if err != nil {
log.Warn().Err(err).Str("topic", *raw.TopicPartition.Topic).Send()
continue
}
// Store users
_ = repo.Store(context.Background(),
&domain.User{
ID: msg.SellerID,
Castle: msg.SellerCastle,
Name: msg.SellerName,
Source: "deals",
},
)
_ = repo.Store(context.Background(),
&domain.User{
ID: msg.BuyerID,
Castle: msg.BuyerCastle,
Name: msg.BuyerName,
Source: "deals",
},
)
}
}
}

26
go.mod

@ -0,0 +1,26 @@
module gitea.russia9.dev/Russia9/chatwars-spy
go 1.18
require (
github.com/confluentinc/confluent-kafka-go v1.8.2
github.com/google/uuid v1.3.0
github.com/rs/zerolog v1.26.1
go.mongodb.org/mongo-driver v1.9.0
)
require (
github.com/go-stack/stack v1.8.1 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/klauspost/compress v1.15.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/stretchr/testify v1.7.0 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.1.1 // indirect
github.com/xdg-go/stringprep v1.0.3 // indirect
github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/text v0.3.7 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)

96
go.sum

@ -0,0 +1,96 @@
github.com/confluentinc/confluent-kafka-go v1.8.2 h1:PBdbvYpyOdFLehj8j+9ba7FL4c4Moxn79gy9cYKxG5E=
github.com/confluentinc/confluent-kafka-go v1.8.2/go.mod h1:u2zNLny2xq+5rWeTQjFHbDzzNuba4P1vo31r9r4uAdg=
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A=
github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc=
github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk=
github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.mongodb.org/mongo-driver v1.9.0 h1:f3aLGJvQmBl8d9S40IL+jEyBC6hfLPbJjv9t5hEM9ck=
go.mongodb.org/mongo-driver v1.9.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA=
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

10
internal/utils/common.go

@ -0,0 +1,10 @@
package utils
import "os"
func GetEnv(key, fallback string) string {
if value, ok := os.LookupEnv(key); ok {
return value
}
return fallback
}

22
pkg/domain/user.go

@ -0,0 +1,22 @@
package domain
import (
"context"
"time"
)
type User struct {
ID string `bson:"_id"`
Castle string
Guild string
Name string
Level int
Source string
FirstSeen time.Time
LastSeen time.Time
}
type UserRepo interface {
Store(ctx context.Context, object *User) error
}

13
pkg/message/deal.go

@ -0,0 +1,13 @@
package message
type Deal struct {
SellerID string `json:"sellerId"`
SellerCastle string `json:"sellerCastle"`
SellerName string `json:"sellerName"`
BuyerID string `json:"buyerId"`
BuyerCastle string `json:"buyerCastle"`
BuyerName string `json:"buyerName"`
Item string `json:"item"`
Quantity int `json:"qty"`
Price int `json:"price"`
}

17
pkg/message/duel.go

@ -0,0 +1,17 @@
package message
type Duel struct {
Winner DuelUser `json:"winner"`
Loser DuelUser `json:"loser"`
IsChallenge bool `json:"isChallenge"`
IsGuildDuel bool `json:"isGuildDuel"`
}
type DuelUser struct {
ID string `json:"id"`
Name string `json:"name"`
Tag string `json:"tag,omitempty"`
Castle string `json:"castle"`
Level int `json:"level"`
Health int `json:"hp"`
}

20
pkg/message/lot.go

@ -0,0 +1,20 @@
package message
type Lot struct {
LotID string `json:"lotId"`
ItemName string `json:"itemName"`
SellerTag string `json:"sellerTag"`
SellerName string `json:"sellerName"`
Quality string `json:"quality"`
SellerCastle string `json:"sellerCastle"`
Condition string `json:"condition"`
EndAt string `json:"endAt"`
StartAt string `json:"startedAt"`
BuyerCastle string `json:"buyerCastle"`
Status string `json:"status"`
FinishedAt string `json:"finishedAt"`
BuyerTag string `json:"buyerTag"`
BuyerName string `json:"buyerName"`
Price int `json:"price"`
Stats map[string]int
}

10
pkg/message/offer.go

@ -0,0 +1,10 @@
package message
type Offer struct {
SellerID string `json:"sellerId"`
SellerName string `json:"sellerName"`
SellerCastle string `json:"sellerCastle"`
Item string `json:"item"`
Quantity int `json:"qty"`
Price int `json:"price"`
}

23
pkg/message/store.go

@ -0,0 +1,23 @@
package message
type Store struct {
Link string `json:"link"`
Name string `json:"name"`
OwnerName string `json:"ownerName"`
OwnerCastle string `json:"ownerCastle"`
Kind string `json:"kind"`
Mana int `json:"mana"`
Offers []StoreOffer `json:"offers"`
Specialization map[string]int `json:"specialization,omitempty"`
QualityCraftLevel int `json:"qualityCraftLevel"`
MaintenanceEnabled bool `json:"maintenanceEnabled"`
MaintenanceCost int `json:"maintenanceCost"`
GuildDiscount int `json:"guildDiscount"`
CastleDiscount int `json:"castleDiscount"`
}
type StoreOffer struct {
Item string `json:"item"`
Price int `json:"price"`
Mana int `json:"mana"`
}

52
pkg/storage/mongodb/user.go

@ -0,0 +1,52 @@
package mongodb
import (
"context"
"errors"
"gitea.russia9.dev/Russia9/chatwars-spy/pkg/domain"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"time"
)
func NewUserRepo(database *mongo.Database) domain.UserRepo {
return &userRepo{Collection: database.Collection("users")}
}
type userRepo struct {
Collection *mongo.Collection
}
func (r *userRepo) Store(ctx context.Context, object *domain.User) error {
cursor := r.Collection.FindOne(ctx, bson.M{"_id": object.ID})
if errors.Is(cursor.Err(), mongo.ErrNoDocuments) {
object.FirstSeen = time.Now()
object.LastSeen = time.Now()
_, err := r.Collection.InsertOne(ctx, object)
return err
} else if cursor.Err() != nil {
return cursor.Err()
}
// Get old user
var old domain.User
err := cursor.Decode(&old)
if err != nil {
return err
}
// Get old guild if empty
if object.Guild == "" {
object.Guild = old.Guild
}
// Set old source
object.Source = old.Source
// Set LastSeen date
object.LastSeen = time.Now()
// Update object in DB
_, err = r.Collection.ReplaceOne(ctx, bson.M{"_id": object.ID}, object)
return err
}

202
vendor/github.com/confluentinc/confluent-kafka-go/LICENSE generated vendored

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

2
vendor/github.com/confluentinc/confluent-kafka-go/kafka/.gitignore generated vendored

@ -0,0 +1,2 @@
testconf.json
go_rdkafka_generr/go_rdkafka_generr

58
vendor/github.com/confluentinc/confluent-kafka-go/kafka/00version.go generated vendored

@ -0,0 +1,58 @@
package kafka
/**
* Copyright 2016-2019 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import (
"fmt"
)
/*
#include "select_rdkafka.h"
//Minimum required librdkafka version. This is checked both during
//build-time and runtime.
//Make sure to keep the MIN_RD_KAFKA_VERSION, MIN_VER_ERRSTR and #error
//defines and strings in sync.
//
#define MIN_RD_KAFKA_VERSION 0x01060000
#ifdef __APPLE__
#define MIN_VER_ERRSTR "confluent-kafka-go requires librdkafka v1.6.0 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`"
#else
#define MIN_VER_ERRSTR "confluent-kafka-go requires librdkafka v1.6.0 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html"
#endif
#if RD_KAFKA_VERSION < MIN_RD_KAFKA_VERSION
#ifdef __APPLE__
#error "confluent-kafka-go requires librdkafka v1.6.0 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`"
#else
#error "confluent-kafka-go requires librdkafka v1.6.0 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html"
#endif
#endif
*/
import "C"
func versionCheck() error {
ver, verstr := LibraryVersion()
if ver < C.MIN_RD_KAFKA_VERSION {
return newErrorFromString(ErrNotImplemented,
fmt.Sprintf("%s: librdkafka version %s (0x%x) detected",
C.MIN_VER_ERRSTR, verstr, ver))
}
return nil
}

145
vendor/github.com/confluentinc/confluent-kafka-go/kafka/README.md generated vendored

@ -0,0 +1,145 @@
# Information for confluent-kafka-go developers
Whenever librdkafka error codes are updated make sure to run generate
before building:
```
$ make -f mk/Makefile generr
$ go build ./...
```
## Testing
Some of the tests included in this directory, the benchmark and integration tests in particular,
require an existing Kafka cluster and a testconf.json configuration file to
provide tests with bootstrap brokers, topic name, etc.
The format of testconf.json is a JSON object:
```
{
"Brokers": "<bootstrap-brokers>",
"Topic": "<test-topic-name>"
}
```
See testconf-example.json for an example and full set of available options.
To run unit-tests:
```
$ go test
```
To run benchmark tests:
```
$ go test -bench .
```
For the code coverage:
```
$ go test -coverprofile=coverage.out -bench=.
$ go tool cover -func=coverage.out
```
## Build tags
Different build types are supported through Go build tags (`-tags ..`),
these tags should be specified on the **application** build/get/install command.
* By default the bundled platform-specific static build of librdkafka will
be used. This works out of the box on Mac OSX and glibc-based Linux distros,
such as Ubuntu and CentOS.
* `-tags musl` - must be specified when building on/for musl-based Linux
distros, such as Alpine. Will use the bundled static musl build of
librdkafka.
* `-tags dynamic` - link librdkafka dynamically. A shared librdkafka library
must be installed manually through other means (apt-get, yum, build from
source, etc).
## Generating HTML documentation
To generate one-page HTML documentation run the mk/doc-gen.py script from the
top-level directory. This script requires the beautifulsoup4 Python package.
```
$ source .../your/virtualenv/bin/activate
$ pip install beautifulsoup4
...
$ make -f mk/Makefile docs
```
## Release process
For each release candidate and final release, perform the following steps:
### Update bundle to latest librdkafka
See instructions in [kafka/librdkafka/README.md](kafka/librdkafka/README.md).
### Update librdkafka version requirement
Update the minimum required librdkafka version in `kafka/00version.go`
and `README.md`.
### Update error codes
Error codes can be automatically generated from the current librdkafka version.
Update generated error codes:
$ make -f mk/Makefile generr
# Verify by building
### Rebuild everything
$ go clean -i ./...
$ go build ./...
### Run full test suite
Set up a test cluster using whatever mechanism you typically use
(docker, trivup, ccloud, ..).
Make sure to update `kafka/testconf.json` as needed (broker list, $BROKERS)
Run test suite:
$ go test ./...
### Verify examples
Manually verify that the examples/ applications work.
Also make sure the examples in README.md work.
Convert any examples using `github.com/confluentinc/confluent-kafka-go/kafka` to use
`gopkg.in/confluentinc/confluent-kafka-go.v1/kafka` import path.
$ find examples/ -type f -name *\.go -exec sed -i -e 's|github\.com/confluentinc/confluent-kafka-go/kafka|gopkg\.in/confluentinc/confluent-kafka-go\.v1/kafka|g' {} +
### Commit any changes
Make sure to push to github before creating the tag to have CI tests pass.
### Create and push tag
$ git tag v1.3.0
$ git push --dry-run origin v1.3.0
# Remove --dry-run and re-execute if it looks ok.
### Create release notes page on github

1025
vendor/github.com/confluentinc/confluent-kafka-go/kafka/adminapi.go generated vendored

File diff suppressed because it is too large Load Diff

264
vendor/github.com/confluentinc/confluent-kafka-go/kafka/adminoptions.go generated vendored

@ -0,0 +1,264 @@
/**
* Copyright 2018 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package kafka
import (
"fmt"
"time"
"unsafe"
)
/*
#include "select_rdkafka.h"
#include <stdlib.h>
*/
import "C"
// AdminOptionOperationTimeout sets the broker's operation timeout, such as the
// timeout for CreateTopics to complete the creation of topics on the controller
// before returning a result to the application.
//
// CreateTopics, DeleteTopics, CreatePartitions:
// a value 0 will return immediately after triggering topic
// creation, while > 0 will wait this long for topic creation to propagate
// in cluster.
//
// Default: 0 (return immediately).
//
// Valid for CreateTopics, DeleteTopics, CreatePartitions.
type AdminOptionOperationTimeout struct {
isSet bool
val time.Duration
}
func (ao AdminOptionOperationTimeout) supportsCreateTopics() {
}
func (ao AdminOptionOperationTimeout) supportsDeleteTopics() {
}
func (ao AdminOptionOperationTimeout) supportsCreatePartitions() {
}
func (ao AdminOptionOperationTimeout) apply(cOptions *C.rd_kafka_AdminOptions_t) error {
if !ao.isSet {
return nil
}
cErrstrSize := C.size_t(512)
cErrstr := (*C.char)(C.malloc(cErrstrSize))
defer C.free(unsafe.Pointer(cErrstr))
cErr := C.rd_kafka_AdminOptions_set_operation_timeout(
cOptions, C.int(durationToMilliseconds(ao.val)),
cErrstr, cErrstrSize)
if cErr != 0 {
C.rd_kafka_AdminOptions_destroy(cOptions)
return newCErrorFromString(cErr,
fmt.Sprintf("Failed to set operation timeout: %s", C.GoString(cErrstr)))
}
return nil
}
// SetAdminOperationTimeout sets the broker's operation timeout, such as the
// timeout for CreateTopics to complete the creation of topics on the controller
// before returning a result to the application.
//
// CreateTopics, DeleteTopics, CreatePartitions:
// a value 0 will return immediately after triggering topic
// creation, while > 0 will wait this long for topic creation to propagate
// in cluster.
//
// Default: 0 (return immediately).
//
// Valid for CreateTopics, DeleteTopics, CreatePartitions.
func SetAdminOperationTimeout(t time.Duration) (ao AdminOptionOperationTimeout) {
ao.isSet = true
ao.val = t
return ao
}
// AdminOptionRequestTimeout sets the overall request timeout, including broker
// lookup, request transmission, operation time on broker, and response.
//
// Default: `socket.timeout.ms`.
//
// Valid for all Admin API methods.
type AdminOptionRequestTimeout struct {
isSet bool
val time.Duration
}
func (ao AdminOptionRequestTimeout) supportsCreateTopics() {
}
func (ao AdminOptionRequestTimeout) supportsDeleteTopics() {
}
func (ao AdminOptionRequestTimeout) supportsCreatePartitions() {
}
func (ao AdminOptionRequestTimeout) supportsAlterConfigs() {
}
func (ao AdminOptionRequestTimeout) supportsDescribeConfigs() {
}
func (ao AdminOptionRequestTimeout) apply(cOptions *C.rd_kafka_AdminOptions_t) error {
if !ao.isSet {
return nil
}
cErrstrSize := C.size_t(512)
cErrstr := (*C.char)(C.malloc(cErrstrSize))
defer C.free(unsafe.Pointer(cErrstr))
cErr := C.rd_kafka_AdminOptions_set_request_timeout(
cOptions, C.int(durationToMilliseconds(ao.val)),
cErrstr, cErrstrSize)
if cErr != 0 {
C.rd_kafka_AdminOptions_destroy(cOptions)
return newCErrorFromString(cErr,
fmt.Sprintf("%s", C.GoString(cErrstr)))
}
return nil
}
// SetAdminRequestTimeout sets the overall request timeout, including broker
// lookup, request transmission, operation time on broker, and response.
//
// Default: `socket.timeout.ms`.
//
// Valid for all Admin API methods.
func SetAdminRequestTimeout(t time.Duration) (ao AdminOptionRequestTimeout) {
ao.isSet = true
ao.val = t
return ao
}
// AdminOptionValidateOnly tells the broker to only validate the request,
// without performing the requested operation (create topics, etc).
//
// Default: false.
//
// Valid for CreateTopics, CreatePartitions, AlterConfigs
type AdminOptionValidateOnly struct {
isSet bool
val bool
}
func (ao AdminOptionValidateOnly) supportsCreateTopics() {
}
func (ao AdminOptionValidateOnly) supportsCreatePartitions() {
}
func (ao AdminOptionValidateOnly) supportsAlterConfigs() {
}
func (ao AdminOptionValidateOnly) apply(cOptions *C.rd_kafka_AdminOptions_t) error {
if !ao.isSet {
return nil
}
cErrstrSize := C.size_t(512)
cErrstr := (*C.char)(C.malloc(cErrstrSize))
defer C.free(unsafe.Pointer(cErrstr))
cErr := C.rd_kafka_AdminOptions_set_validate_only(
cOptions, bool2cint(ao.val),
cErrstr, cErrstrSize)
if cErr != 0 {
C.rd_kafka_AdminOptions_destroy(cOptions)
return newCErrorFromString(cErr,
fmt.Sprintf("%s", C.GoString(cErrstr)))
}
return nil
}
// SetAdminValidateOnly tells the broker to only validate the request,
// without performing the requested operation (create topics, etc).
//
// Default: false.
//
// Valid for CreateTopics, DeleteTopics, CreatePartitions, AlterConfigs
func SetAdminValidateOnly(validateOnly bool) (ao AdminOptionValidateOnly) {
ao.isSet = true
ao.val = validateOnly
return ao
}
// CreateTopicsAdminOption - see setters.
//
// See SetAdminRequestTimeout, SetAdminOperationTimeout, SetAdminValidateOnly.
type CreateTopicsAdminOption interface {
supportsCreateTopics()
apply(cOptions *C.rd_kafka_AdminOptions_t) error
}
// DeleteTopicsAdminOption - see setters.
//
// See SetAdminRequestTimeout, SetAdminOperationTimeout.
type DeleteTopicsAdminOption interface {
supportsDeleteTopics()
apply(cOptions *C.rd_kafka_AdminOptions_t) error
}
// CreatePartitionsAdminOption - see setters.
//
// See SetAdminRequestTimeout, SetAdminOperationTimeout, SetAdminValidateOnly.
type CreatePartitionsAdminOption interface {
supportsCreatePartitions()
apply(cOptions *C.rd_kafka_AdminOptions_t) error
}
// AlterConfigsAdminOption - see setters.
//
// See SetAdminRequestTimeout, SetAdminValidateOnly, SetAdminIncremental.
type AlterConfigsAdminOption interface {
supportsAlterConfigs()
apply(cOptions *C.rd_kafka_AdminOptions_t) error
}
// DescribeConfigsAdminOption - see setters.
//
// See SetAdminRequestTimeout.
type DescribeConfigsAdminOption interface {
supportsDescribeConfigs()
apply(cOptions *C.rd_kafka_AdminOptions_t) error
}
// AdminOption is a generic type not to be used directly.
//
// See CreateTopicsAdminOption et.al.
type AdminOption interface {
apply(cOptions *C.rd_kafka_AdminOptions_t) error
}
func adminOptionsSetup(h *handle, opType C.rd_kafka_admin_op_t, options []AdminOption) (*C.rd_kafka_AdminOptions_t, error) {
cOptions := C.rd_kafka_AdminOptions_new(h.rk, opType)
for _, opt := range options {
if opt == nil {
continue
}
err := opt.apply(cOptions)
if err != nil {
return nil, err
}
}
return cOptions, nil
}

4737
vendor/github.com/confluentinc/confluent-kafka-go/kafka/api.html generated vendored

File diff suppressed because it is too large Load Diff

13
vendor/github.com/confluentinc/confluent-kafka-go/kafka/build_darwin.go generated vendored

@ -0,0 +1,13 @@
// +build !dynamic
// This file was auto-generated by librdkafka_vendor/bundle-import.sh, DO NOT EDIT.
package kafka
// #cgo CFLAGS: -DUSE_VENDORED_LIBRDKAFKA -DLIBRDKAFKA_STATICLIB
// #cgo LDFLAGS: ${SRCDIR}/librdkafka_vendor/librdkafka_darwin.a -lm -lsasl2 -ldl -lpthread
import "C"
// LibrdkafkaLinkInfo explains how librdkafka was linked to the Go client
const LibrdkafkaLinkInfo = "static darwin from librdkafka-static-bundle-v1.8.2.tgz"

9
vendor/github.com/confluentinc/confluent-kafka-go/kafka/build_dynamic.go generated vendored

@ -0,0 +1,9 @@
// +build dynamic
package kafka
// #cgo pkg-config: rdkafka
import "C"
// LibrdkafkaLinkInfo explains how librdkafka was linked to the Go client
const LibrdkafkaLinkInfo = "dynamically linked to librdkafka"

13
vendor/github.com/confluentinc/confluent-kafka-go/kafka/build_glibc_linux.go generated vendored

@ -0,0 +1,13 @@
// +build !dynamic
// +build !musl
// This file was auto-generated by librdkafka_vendor/bundle-import.sh, DO NOT EDIT.
package kafka
// #cgo CFLAGS: -DUSE_VENDORED_LIBRDKAFKA -DLIBRDKAFKA_STATICLIB
// #cgo LDFLAGS: ${SRCDIR}/librdkafka_vendor/librdkafka_glibc_linux.a -lm -ldl -lpthread -lrt
import "C"
// LibrdkafkaLinkInfo explains how librdkafka was linked to the Go client
const LibrdkafkaLinkInfo = "static glibc_linux from librdkafka-static-bundle-v1.8.2.tgz"

13
vendor/github.com/confluentinc/confluent-kafka-go/kafka/build_musl_linux.go generated vendored

@ -0,0 +1,13 @@
// +build !dynamic
// +build musl
// This file was auto-generated by librdkafka_vendor/bundle-import.sh, DO NOT EDIT.
package kafka
// #cgo CFLAGS: -DUSE_VENDORED_LIBRDKAFKA -DLIBRDKAFKA_STATICLIB
// #cgo LDFLAGS: ${SRCDIR}/librdkafka_vendor/librdkafka_musl_linux.a -lm -ldl -lpthread -lrt -lpthread -lrt
import "C"
// LibrdkafkaLinkInfo explains how librdkafka was linked to the Go client
const LibrdkafkaLinkInfo = "static musl_linux from librdkafka-static-bundle-v1.8.2.tgz"

13
vendor/github.com/confluentinc/confluent-kafka-go/kafka/build_windows.go generated vendored

@ -0,0 +1,13 @@
// +build !dynamic
// This file was auto-generated by librdkafka_vendor/bundle-import.sh, DO NOT EDIT.
package kafka
// #cgo CFLAGS: -DUSE_VENDORED_LIBRDKAFKA -DLIBRDKAFKA_STATICLIB
// #cgo LDFLAGS: ${SRCDIR}/librdkafka_vendor/librdkafka_windows.a -lws2_32 -lsecur32 -lcrypt32
import "C"
// LibrdkafkaLinkInfo explains how librdkafka was linked to the Go client
const LibrdkafkaLinkInfo = "static windows from librdkafka-static-bundle-v1.8.2.tgz"

280
vendor/github.com/confluentinc/confluent-kafka-go/kafka/config.go generated vendored

@ -0,0 +1,280 @@
package kafka
/**
* Copyright 2016 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import (
"fmt"
"reflect"
"strings"
"unsafe"
)
/*
#include <stdlib.h>
#include "select_rdkafka.h"
*/
import "C"
// ConfigValue supports the following types:
// bool, int, string, any type with the standard String() interface
type ConfigValue interface{}
// ConfigMap is a map containing standard librdkafka configuration properties as documented in:
// https://github.com/edenhill/librdkafka/tree/master/CONFIGURATION.md
//
// The special property "default.topic.config" (optional) is a ConfigMap
// containing default topic configuration properties.
//
// The use of "default.topic.config" is deprecated,
// topic configuration properties shall be specified in the standard ConfigMap.
// For backwards compatibility, "default.topic.config" (if supplied)
// takes precedence.
type ConfigMap map[string]ConfigValue
// SetKey sets configuration property key to value.
//
// For user convenience a key prefixed with {topic}. will be
// set on the "default.topic.config" sub-map, this use is deprecated.
func (m ConfigMap) SetKey(key string, value ConfigValue) error {
if strings.HasPrefix(key, "{topic}.") {
_, found := m["default.topic.config"]
if !found {
m["default.topic.config"] = ConfigMap{}
}
m["default.topic.config"].(ConfigMap)[strings.TrimPrefix(key, "{topic}.")] = value
} else {
m[key] = value
}
return nil
}
// Set implements flag.Set (command line argument parser) as a convenience
// for `-X key=value` config.
func (m ConfigMap) Set(kv string) error {
i := strings.Index(kv, "=")
if i == -1 {
return newErrorFromString(ErrInvalidArg, "Expected key=value")
}
k := kv[:i]
v := kv[i+1:]
return m.SetKey(k, v)
}
func value2string(v ConfigValue) (ret string, errstr string) {
switch x := v.(type) {
case bool:
if x {
ret = "true"
} else {
ret = "false"
}
case int:
ret = fmt.Sprintf("%d", x)
case string:
ret = x
case fmt.Stringer:
ret = x.String()
default:
return "", fmt.Sprintf("Invalid value type %T", v)
}
return ret, ""
}
// rdkAnyconf abstracts rd_kafka_conf_t and rd_kafka_topic_conf_t
// into a common interface.
type rdkAnyconf interface {
set(cKey *C.char, cVal *C.char, cErrstr *C.char, errstrSize int) C.rd_kafka_conf_res_t
}
func anyconfSet(anyconf rdkAnyconf, key string, val ConfigValue) (err error) {
value, errstr := value2string(val)
if errstr != "" {
return newErrorFromString(ErrInvalidArg, fmt.Sprintf("%s for key %s (expected string,bool,int,ConfigMap)", errstr, key))
}
cKey := C.CString(key)
defer C.free(unsafe.Pointer(cKey))
cVal := C.CString(value)
defer C.free(unsafe.Pointer(cVal))
cErrstr := (*C.char)(C.malloc(C.size_t(128)))
defer C.free(unsafe.Pointer(cErrstr))
if anyconf.set(cKey, cVal, cErrstr, 128) != C.RD_KAFKA_CONF_OK {
return newErrorFromCString(C.RD_KAFKA_RESP_ERR__INVALID_ARG, cErrstr)
}
return nil
}
// we need these typedefs to workaround a crash in golint
// when parsing the set() methods below
type rdkConf C.rd_kafka_conf_t
type rdkTopicConf C.rd_kafka_topic_conf_t
func (cConf *rdkConf) set(cKey *C.char, cVal *C.char, cErrstr *C.char, errstrSize int) C.rd_kafka_conf_res_t {
return C.rd_kafka_conf_set((*C.rd_kafka_conf_t)(cConf), cKey, cVal, cErrstr, C.size_t(errstrSize))
}
func (ctopicConf *rdkTopicConf) set(cKey *C.char, cVal *C.char, cErrstr *C.char, errstrSize int) C.rd_kafka_conf_res_t {
return C.rd_kafka_topic_conf_set((*C.rd_kafka_topic_conf_t)(ctopicConf), cKey, cVal, cErrstr, C.size_t(errstrSize))
}
func configConvertAnyconf(m ConfigMap, anyconf rdkAnyconf) (err error) {
// set plugins first, any plugin-specific configuration depends on
// the plugin to have already been set
pluginPaths, ok := m["plugin.library.paths"]
if ok {
err = anyconfSet(anyconf, "plugin.library.paths", pluginPaths)
if err != nil {
return err
}
}
for k, v := range m {
if k == "plugin.library.paths" {
continue
}
switch v.(type) {
case ConfigMap:
/* Special sub-ConfigMap, only used for default.topic.config */
if k != "default.topic.config" {
return newErrorFromString(ErrInvalidArg, fmt.Sprintf("Invalid type for key %s", k))
}
var cTopicConf = C.rd_kafka_topic_conf_new()
err = configConvertAnyconf(v.(ConfigMap),
(*rdkTopicConf)(cTopicConf))
if err != nil {
C.rd_kafka_topic_conf_destroy(cTopicConf)
return err
}
C.rd_kafka_conf_set_default_topic_conf(
(*C.rd_kafka_conf_t)(anyconf.(*rdkConf)),
(*C.rd_kafka_topic_conf_t)((*rdkTopicConf)(cTopicConf)))
default:
err = anyconfSet(anyconf, k, v)
if err != nil {
return err
}
}
}
return nil
}
// convert ConfigMap to C rd_kafka_conf_t *
func (m ConfigMap) convert() (cConf *C.rd_kafka_conf_t, err error) {
cConf = C.rd_kafka_conf_new()
// Set the client.software.name and .version (use librdkafka version).
_, librdkafkaVersion := LibraryVersion()
anyconfSet((*rdkConf)(cConf), "client.software.name", "confluent-kafka-go")
anyconfSet((*rdkConf)(cConf), "client.software.version", librdkafkaVersion)
err = configConvertAnyconf(m, (*rdkConf)(cConf))
if err != nil {
C.rd_kafka_conf_destroy(cConf)
return nil, err
}
return cConf, nil
}
// get finds key in the configmap and returns its value.
// If the key is not found defval is returned.
// If the key is found but the type is mismatched an error is returned.
func (m ConfigMap) get(key string, defval ConfigValue) (ConfigValue, error) {
if strings.HasPrefix(key, "{topic}.") {
defconfCv, found := m["default.topic.config"]
if !found {
return defval, nil
}
return defconfCv.(ConfigMap).get(strings.TrimPrefix(key, "{topic}."), defval)
}
v, ok := m[key]
if !ok {
return defval, nil
}
if defval != nil && reflect.TypeOf(defval) != reflect.TypeOf(v) {
return nil, newErrorFromString(ErrInvalidArg, fmt.Sprintf("%s expects type %T, not %T", key, defval, v))
}
return v, nil
}
// extract performs a get() and if found deletes the key.
func (m ConfigMap) extract(key string, defval ConfigValue) (ConfigValue, error) {
v, err := m.get(key, defval)
if err != nil {
return nil, err
}
delete(m, key)
return v, nil
}
// extractLogConfig extracts generic go.logs.* configuration properties.
func (m ConfigMap) extractLogConfig() (logsChanEnable bool, logsChan chan LogEvent, err error) {
v, err := m.extract("go.logs.channel.enable", false)
if err != nil {
return
}
logsChanEnable = v.(bool)
v, err = m.extract("go.logs.channel", nil)
if err != nil {
return
}
if v != nil {
logsChan = v.(chan LogEvent)
}
if logsChanEnable {
// Tell librdkafka to forward logs to the log queue
m.Set("log.queue=true")
}
return
}
func (m ConfigMap) clone() ConfigMap {
m2 := make(ConfigMap)
for k, v := range m {
m2[k] = v
}
return m2
}
// Get finds the given key in the ConfigMap and returns its value.
// If the key is not found `defval` is returned.
// If the key is found but the type does not match that of `defval` (unless nil)
// an ErrInvalidArg error is returned.
func (m ConfigMap) Get(key string, defval ConfigValue) (ConfigValue, error) {
return m.get(key, defval)
}

923
vendor/github.com/confluentinc/confluent-kafka-go/kafka/consumer.go generated vendored

@ -0,0 +1,923 @@
package kafka
/**
* Copyright 2016-2020 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import (
"fmt"
"math"
"time"
"unsafe"
)
/*
#include <stdlib.h>
#include "select_rdkafka.h"
static rd_kafka_topic_partition_t *_c_rdkafka_topic_partition_list_entry(rd_kafka_topic_partition_list_t *rktparlist, int idx) {
return idx < rktparlist->cnt ? &rktparlist->elems[idx] : NULL;
}
*/
import "C"
// RebalanceCb provides a per-Subscribe*() rebalance event callback.
// The passed Event will be either AssignedPartitions or RevokedPartitions
type RebalanceCb func(*Consumer, Event) error
// Consumer implements a High-level Apache Kafka Consumer instance
type Consumer struct {
events chan Event
handle handle
eventsChanEnable bool
readerTermChan chan bool
rebalanceCb RebalanceCb
appReassigned bool
appRebalanceEnable bool // Config setting
}
// Strings returns a human readable name for a Consumer instance
func (c *Consumer) String() string {
return c.handle.String()
}
// getHandle implements the Handle interface
func (c *Consumer) gethandle() *handle {
return &c.handle
}
// Subscribe to a single topic
// This replaces the current subscription
func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error {
return c.SubscribeTopics([]string{topic}, rebalanceCb)
}
// SubscribeTopics subscribes to the provided list of topics.
// This replaces the current subscription.
func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) (err error) {
ctopics := C.rd_kafka_topic_partition_list_new(C.int(len(topics)))
defer C.rd_kafka_topic_partition_list_destroy(ctopics)
for _, topic := range topics {
ctopic := C.CString(topic)
defer C.free(unsafe.Pointer(ctopic))
C.rd_kafka_topic_partition_list_add(ctopics, ctopic, C.RD_KAFKA_PARTITION_UA)
}
e := C.rd_kafka_subscribe(c.handle.rk, ctopics)
if e != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return newError(e)
}
c.rebalanceCb = rebalanceCb
return nil
}
// Unsubscribe from the current subscription, if any.
func (c *Consumer) Unsubscribe() (err error) {
C.rd_kafka_unsubscribe(c.handle.rk)
return nil
}
// Assign an atomic set of partitions to consume.
//
// The .Offset field of each TopicPartition must either be set to an absolute
// starting offset (>= 0), or one of the logical offsets (`kafka.OffsetEnd` etc),
// but should typically be set to `kafka.OffsetStored` to have the consumer
// use the committed offset as a start position, with a fallback to
// `auto.offset.reset` if there is no committed offset.
//
// This replaces the current assignment.
func (c *Consumer) Assign(partitions []TopicPartition) (err error) {
c.appReassigned = true
cparts := newCPartsFromTopicPartitions(partitions)
defer C.rd_kafka_topic_partition_list_destroy(cparts)
e := C.rd_kafka_assign(c.handle.rk, cparts)
if e != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return newError(e)
}
return nil
}
// Unassign the current set of partitions to consume.
func (c *Consumer) Unassign() (err error) {
c.appReassigned = true
e := C.rd_kafka_assign(c.handle.rk, nil)
if e != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return newError(e)
}
return nil
}
// IncrementalAssign adds the specified partitions to the current set of
// partitions to consume.
//
// The .Offset field of each TopicPartition must either be set to an absolute
// starting offset (>= 0), or one of the logical offsets (`kafka.OffsetEnd` etc),
// but should typically be set to `kafka.OffsetStored` to have the consumer
// use the committed offset as a start position, with a fallback to
// `auto.offset.reset` if there is no committed offset.
//
// The new partitions must not be part of the current assignment.
func (c *Consumer) IncrementalAssign(partitions []TopicPartition) (err error) {
c.appReassigned = true
cparts := newCPartsFromTopicPartitions(partitions)
defer C.rd_kafka_topic_partition_list_destroy(cparts)
cError := C.rd_kafka_incremental_assign(c.handle.rk, cparts)
if cError != nil {
return newErrorFromCErrorDestroy(cError)
}
return nil
}
// IncrementalUnassign removes the specified partitions from the current set of
// partitions to consume.
//
// The .Offset field of the TopicPartition is ignored.
//
// The removed partitions must be part of the current assignment.
func (c *Consumer) IncrementalUnassign(partitions []TopicPartition) (err error) {
c.appReassigned = true
cparts := newCPartsFromTopicPartitions(partitions)
defer C.rd_kafka_topic_partition_list_destroy(cparts)
cError := C.rd_kafka_incremental_unassign(c.handle.rk, cparts)
if cError != nil {
return newErrorFromCErrorDestroy(cError)
}
return nil
}
// GetRebalanceProtocol returns the current consumer group rebalance protocol,
// which is either "EAGER" or "COOPERATIVE".
// If the rebalance protocol is not known in the current state an empty string
// is returned.
// Should typically only be called during rebalancing.
func (c *Consumer) GetRebalanceProtocol() string {
cStr := C.rd_kafka_rebalance_protocol(c.handle.rk)
if cStr == nil {
return ""
}
return C.GoString(cStr)
}
// AssignmentLost returns true if current partition assignment has been lost.
// This method is only applicable for use with a subscribing consumer when
// handling a rebalance event or callback.
// Partitions that have been lost may already be owned by other members in the
// group and therefore commiting offsets, for example, may fail.
func (c *Consumer) AssignmentLost() bool {
return cint2bool(C.rd_kafka_assignment_lost(c.handle.rk))
}
// commit offsets for specified offsets.
// If offsets is nil the currently assigned partitions' offsets are committed.
// This is a blocking call, caller will need to wrap in go-routine to
// get async or throw-away behaviour.
func (c *Consumer) commit(offsets []TopicPartition) (committedOffsets []TopicPartition, err error) {
var rkqu *C.rd_kafka_queue_t
rkqu = C.rd_kafka_queue_new(c.handle.rk)
defer C.rd_kafka_queue_destroy(rkqu)
var coffsets *C.rd_kafka_topic_partition_list_t
if offsets != nil {
coffsets = newCPartsFromTopicPartitions(offsets)
defer C.rd_kafka_topic_partition_list_destroy(coffsets)
}
cErr := C.rd_kafka_commit_queue(c.handle.rk, coffsets, rkqu, nil, nil)
if cErr != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return nil, newError(cErr)
}
rkev := C.rd_kafka_queue_poll(rkqu, C.int(-1))
if rkev == nil {
// shouldn't happen
return nil, newError(C.RD_KAFKA_RESP_ERR__DESTROY)
}
defer C.rd_kafka_event_destroy(rkev)
if C.rd_kafka_event_type(rkev) != C.RD_KAFKA_EVENT_OFFSET_COMMIT {
panic(fmt.Sprintf("Expected OFFSET_COMMIT, got %s",
C.GoString(C.rd_kafka_event_name(rkev))))
}
cErr = C.rd_kafka_event_error(rkev)
if cErr != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return nil, newErrorFromCString(cErr, C.rd_kafka_event_error_string(rkev))
}
cRetoffsets := C.rd_kafka_event_topic_partition_list(rkev)
if cRetoffsets == nil {
// no offsets, no error
return nil, nil
}
committedOffsets = newTopicPartitionsFromCparts(cRetoffsets)
return committedOffsets, nil
}
// Commit offsets for currently assigned partitions
// This is a blocking call.
// Returns the committed offsets on success.
func (c *Consumer) Commit() ([]TopicPartition, error) {
return c.commit(nil)
}
// CommitMessage commits offset based on the provided message.
// This is a blocking call.
// Returns the committed offsets on success.
func (c *Consumer) CommitMessage(m *Message) ([]TopicPartition, error) {
if m.TopicPartition.Error != nil {
return nil, newErrorFromString(ErrInvalidArg, "Can't commit errored message")
}
offsets := []TopicPartition{m.TopicPartition}
offsets[0].Offset++
return c.commit(offsets)
}
// CommitOffsets commits the provided list of offsets
// This is a blocking call.
// Returns the committed offsets on success.
func (c *Consumer) CommitOffsets(offsets []TopicPartition) ([]TopicPartition, error) {
return c.commit(offsets)
}
// StoreOffsets stores the provided list of offsets that will be committed
// to the offset store according to `auto.commit.interval.ms` or manual
// offset-less Commit().
//
// Returns the stored offsets on success. If at least one offset couldn't be stored,
// an error and a list of offsets is returned. Each offset can be checked for
// specific errors via its `.Error` member.
func (c *Consumer) StoreOffsets(offsets []TopicPartition) (storedOffsets []TopicPartition, err error) {
coffsets := newCPartsFromTopicPartitions(offsets)
defer C.rd_kafka_topic_partition_list_destroy(coffsets)
cErr := C.rd_kafka_offsets_store(c.handle.rk, coffsets)
// coffsets might be annotated with an error
storedOffsets = newTopicPartitionsFromCparts(coffsets)
if cErr != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return storedOffsets, newError(cErr)
}
return storedOffsets, nil
}
// StoreMessage stores offset based on the provided message.
// This is a convenience method that uses StoreOffsets to do the actual work.
func (c *Consumer) StoreMessage(m *Message) (storedOffsets []TopicPartition, err error) {
if m.TopicPartition.Error != nil {
return nil, newErrorFromString(ErrInvalidArg, "Can't store errored message")
}
if m.TopicPartition.Offset < 0 {
return nil, newErrorFromString(ErrInvalidArg, "Can't store message with offset less than 0")
}
offsets := []TopicPartition{m.TopicPartition}
offsets[0].Offset++
return c.StoreOffsets(offsets)
}
// Seek seeks the given topic partitions using the offset from the TopicPartition.
//
// If timeoutMs is not 0 the call will wait this long for the
// seek to be performed. If the timeout is reached the internal state
// will be unknown and this function returns ErrTimedOut.
// If timeoutMs is 0 it will initiate the seek but return
// immediately without any error reporting (e.g., async).
//
// Seek() may only be used for partitions already being consumed
// (through Assign() or implicitly through a self-rebalanced Subscribe()).
// To set the starting offset it is preferred to use Assign() and provide
// a starting offset for each partition.
//
// Returns an error on failure or nil otherwise.
func (c *Consumer) Seek(partition TopicPartition, timeoutMs int) error {
rkt := c.handle.getRkt(*partition.Topic)
cErr := C.rd_kafka_seek(rkt,
C.int32_t(partition.Partition),
C.int64_t(partition.Offset),
C.int(timeoutMs))
if cErr != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return newError(cErr)
}
return nil
}
// Poll the consumer for messages or events.
//
// Will block for at most timeoutMs milliseconds
//
// The following callbacks may be triggered:
// Subscribe()'s rebalanceCb
//
// Returns nil on timeout, else an Event
func (c *Consumer) Poll(timeoutMs int) (event Event) {
ev, _ := c.handle.eventPoll(nil, timeoutMs, 1, nil)
return ev
}
// Events returns the Events channel (if enabled)
func (c *Consumer) Events() chan Event {
return c.events
}
// Logs returns the log channel if enabled, or nil otherwise.
func (c *Consumer) Logs() chan LogEvent {
return c.handle.logs
}
// ReadMessage polls the consumer for a message.
//
// This is a convenience API that wraps Poll() and only returns
// messages or errors. All other event types are discarded.
//
// The call will block for at most `timeout` waiting for
// a new message or error. `timeout` may be set to -1 for
// indefinite wait.
//
// Timeout is returned as (nil, err) where err is `err.(kafka.Error).Code() == kafka.ErrTimedOut`.
//
// Messages are returned as (msg, nil),
// while general errors are returned as (nil, err),
// and partition-specific errors are returned as (msg, err) where
// msg.TopicPartition provides partition-specific information (such as topic, partition and offset).
//
// All other event types, such as PartitionEOF, AssignedPartitions, etc, are silently discarded.
//
func (c *Consumer) ReadMessage(timeout time.Duration) (*Message, error) {
var absTimeout time.Time
var timeoutMs int
if timeout > 0 {
absTimeout = time.Now().Add(timeout)
timeoutMs = (int)(timeout.Seconds() * 1000.0)
} else {
timeoutMs = (int)(timeout)
}
for {
ev := c.Poll(timeoutMs)
switch e := ev.(type) {
case *Message:
if e.TopicPartition.Error != nil {
return e, e.TopicPartition.Error
}
return e, nil
case Error:
return nil, e
default:
// Ignore other event types
}
if timeout > 0 {
// Calculate remaining time
timeoutMs = int(math.Max(0.0, absTimeout.Sub(time.Now()).Seconds()*1000.0))
}
if timeoutMs == 0 && ev == nil {
return nil, newError(C.RD_KAFKA_RESP_ERR__TIMED_OUT)
}
}
}
// Close Consumer instance.
// The object is no longer usable after this call.
func (c *Consumer) Close() (err error) {
// Wait for consumerReader() or pollLogEvents to terminate (by closing readerTermChan)
close(c.readerTermChan)
c.handle.waitGroup.Wait()
if c.eventsChanEnable {
close(c.events)
}
// librdkafka's rd_kafka_consumer_close() will block
// and trigger the rebalance_cb() if one is set, if not, which is the
// case with the Go client since it registers EVENTs rather than callbacks,
// librdkafka will shortcut the rebalance_cb and do a forced unassign.
// But we can't have that since the application might need the final RevokePartitions
// before shutting down. So we trigger an Unsubscribe() first, wait for that to
// propagate (in the Poll loop below), and then close the consumer.
c.Unsubscribe()
// Poll for rebalance events
for {
c.Poll(10 * 1000)
if int(C.rd_kafka_queue_length(c.handle.rkq)) == 0 {
break
}
}
// Destroy our queue
C.rd_kafka_queue_destroy(c.handle.rkq)
c.handle.rkq = nil
// Close the consumer
C.rd_kafka_consumer_close(c.handle.rk)
c.handle.cleanup()
C.rd_kafka_destroy(c.handle.rk)
return nil
}
// NewConsumer creates a new high-level Consumer instance.
//
// conf is a *ConfigMap with standard librdkafka configuration properties.
//
// Supported special configuration properties:
// go.application.rebalance.enable (bool, false) - Forward rebalancing responsibility to application via the Events() channel.
// If set to true the app must handle the AssignedPartitions and
// RevokedPartitions events and call Assign() and Unassign()
// respectively.
// go.events.channel.enable (bool, false) - [deprecated] Enable the Events() channel. Messages and events will be pushed on the Events() channel and the Poll() interface will be disabled.
// go.events.channel.size (int, 1000) - Events() channel size
// go.logs.channel.enable (bool, false) - Forward log to Logs() channel.
// go.logs.channel (chan kafka.LogEvent, nil) - Forward logs to application-provided channel instead of Logs(). Requires go.logs.channel.enable=true.
//
// WARNING: Due to the buffering nature of channels (and queues in general) the
// use of the events channel risks receiving outdated events and
// messages. Minimizing go.events.channel.size reduces the risk
// and number of outdated events and messages but does not eliminate
// the factor completely. With a channel size of 1 at most one
// event or message may be outdated.
func NewConsumer(conf *ConfigMap) (*Consumer, error) {
err := versionCheck()
if err != nil {
return nil, err
}
// before we do anything with the configuration, create a copy such that
// the original is not mutated.
confCopy := conf.clone()
groupid, _ := confCopy.get("group.id", nil)
if groupid == nil {
// without a group.id the underlying cgrp subsystem in librdkafka wont get started
// and without it there is no way to consume assigned partitions.
// So for now require the group.id, this might change in the future.
return nil, newErrorFromString(ErrInvalidArg, "Required property group.id not set")
}
c := &Consumer{}
v, err := confCopy.extract("go.application.rebalance.enable", false)
if err != nil {
return nil, err
}
c.appRebalanceEnable = v.(bool)
v, err = confCopy.extract("go.events.channel.enable", false)
if err != nil {
return nil, err
}
c.eventsChanEnable = v.(bool)
v, err = confCopy.extract("go.events.channel.size", 1000)
if err != nil {
return nil, err
}
eventsChanSize := v.(int)
logsChanEnable, logsChan, err := confCopy.extractLogConfig()
if err != nil {
return nil, err
}
cConf, err := confCopy.convert()
if err != nil {
return nil, err
}
cErrstr := (*C.char)(C.malloc(C.size_t(256)))
defer C.free(unsafe.Pointer(cErrstr))
C.rd_kafka_conf_set_events(cConf, C.RD_KAFKA_EVENT_REBALANCE|C.RD_KAFKA_EVENT_OFFSET_COMMIT|C.RD_KAFKA_EVENT_STATS|C.RD_KAFKA_EVENT_ERROR|C.RD_KAFKA_EVENT_OAUTHBEARER_TOKEN_REFRESH)
c.handle.rk = C.rd_kafka_new(C.RD_KAFKA_CONSUMER, cConf, cErrstr, 256)
if c.handle.rk == nil {
return nil, newErrorFromCString(C.RD_KAFKA_RESP_ERR__INVALID_ARG, cErrstr)
}
C.rd_kafka_poll_set_consumer(c.handle.rk)
c.handle.c = c
c.handle.setup()
c.readerTermChan = make(chan bool)
c.handle.rkq = C.rd_kafka_queue_get_consumer(c.handle.rk)
if c.handle.rkq == nil {
// no cgrp (no group.id configured), revert to main queue.
c.handle.rkq = C.rd_kafka_queue_get_main(c.handle.rk)
}
if logsChanEnable {
c.handle.setupLogQueue(logsChan, c.readerTermChan)
}
if c.eventsChanEnable {
c.events = make(chan Event, eventsChanSize)
/* Start rdkafka consumer queue reader -> events writer goroutine */
c.handle.waitGroup.Add(1)
go func() {
consumerReader(c, c.readerTermChan)
c.handle.waitGroup.Done()
}()
}
return c, nil
}
// consumerReader reads messages and events from the librdkafka consumer queue
// and posts them on the consumer channel.
// Runs until termChan closes
func consumerReader(c *Consumer, termChan chan bool) {
for {
select {
case _ = <-termChan:
return
default:
_, term := c.handle.eventPoll(c.events, 100, 1000, termChan)
if term {
return
}
}
}
}
// GetMetadata queries broker for cluster and topic metadata.
// If topic is non-nil only information about that topic is returned, else if
// allTopics is false only information about locally used topics is returned,
// else information about all topics is returned.
// GetMetadata is equivalent to listTopics, describeTopics and describeCluster in the Java API.
func (c *Consumer) GetMetadata(topic *string, allTopics bool, timeoutMs int) (*Metadata, error) {
return getMetadata(c, topic, allTopics, timeoutMs)
}
// QueryWatermarkOffsets queries the broker for the low and high offsets for the given topic and partition.
func (c *Consumer) QueryWatermarkOffsets(topic string, partition int32, timeoutMs int) (low, high int64, err error) {
return queryWatermarkOffsets(c, topic, partition, timeoutMs)
}
// GetWatermarkOffsets returns the cached low and high offsets for the given topic
// and partition. The high offset is populated on every fetch response or via calling QueryWatermarkOffsets.
// The low offset is populated every statistics.interval.ms if that value is set.
// OffsetInvalid will be returned if there is no cached offset for either value.
func (c *Consumer) GetWatermarkOffsets(topic string, partition int32) (low, high int64, err error) {
return getWatermarkOffsets(c, topic, partition)
}
// OffsetsForTimes looks up offsets by timestamp for the given partitions.
//
// The returned offset for each partition is the earliest offset whose
// timestamp is greater than or equal to the given timestamp in the
// corresponding partition. If the provided timestamp exceeds that of the
// last message in the partition, a value of -1 will be returned.
//
// The timestamps to query are represented as `.Offset` in the `times`
// argument and the looked up offsets are represented as `.Offset` in the returned
// `offsets` list.
//
// The function will block for at most timeoutMs milliseconds.
//
// Duplicate Topic+Partitions are not supported.
// Per-partition errors may be returned in the `.Error` field.
func (c *Consumer) OffsetsForTimes(times []TopicPartition, timeoutMs int) (offsets []TopicPartition, err error) {
return offsetsForTimes(c, times, timeoutMs)
}
// Subscription returns the current subscription as set by Subscribe()
func (c *Consumer) Subscription() (topics []string, err error) {
var cTopics *C.rd_kafka_topic_partition_list_t
cErr := C.rd_kafka_subscription(c.handle.rk, &cTopics)
if cErr != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return nil, newError(cErr)
}
defer C.rd_kafka_topic_partition_list_destroy(cTopics)
topicCnt := int(cTopics.cnt)
topics = make([]string, topicCnt)
for i := 0; i < topicCnt; i++ {
crktpar := C._c_rdkafka_topic_partition_list_entry(cTopics,
C.int(i))
topics[i] = C.GoString(crktpar.topic)
}
return topics, nil
}
// Assignment returns the current partition assignments
func (c *Consumer) Assignment() (partitions []TopicPartition, err error) {
var cParts *C.rd_kafka_topic_partition_list_t
cErr := C.rd_kafka_assignment(c.handle.rk, &cParts)
if cErr != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return nil, newError(cErr)
}
defer C.rd_kafka_topic_partition_list_destroy(cParts)
partitions = newTopicPartitionsFromCparts(cParts)
return partitions, nil
}
// Committed retrieves committed offsets for the given set of partitions
func (c *Consumer) Committed(partitions []TopicPartition, timeoutMs int) (offsets []TopicPartition, err error) {
cparts := newCPartsFromTopicPartitions(partitions)
defer C.rd_kafka_topic_partition_list_destroy(cparts)
cerr := C.rd_kafka_committed(c.handle.rk, cparts, C.int(timeoutMs))
if cerr != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return nil, newError(cerr)
}
return newTopicPartitionsFromCparts(cparts), nil
}
// Position returns the current consume position for the given partitions.
// Typical use is to call Assignment() to get the partition list
// and then pass it to Position() to get the current consume position for
// each of the assigned partitions.
// The consume position is the next message to read from the partition.
// i.e., the offset of the last message seen by the application + 1.
func (c *Consumer) Position(partitions []TopicPartition) (offsets []TopicPartition, err error) {
cparts := newCPartsFromTopicPartitions(partitions)
defer C.rd_kafka_topic_partition_list_destroy(cparts)
cerr := C.rd_kafka_position(c.handle.rk, cparts)
if cerr != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return nil, newError(cerr)
}
return newTopicPartitionsFromCparts(cparts), nil
}
// Pause consumption for the provided list of partitions
//
// Note that messages already enqueued on the consumer's Event channel
// (if `go.events.channel.enable` has been set) will NOT be purged by
// this call, set `go.events.channel.size` accordingly.
func (c *Consumer) Pause(partitions []TopicPartition) (err error) {
cparts := newCPartsFromTopicPartitions(partitions)
defer C.rd_kafka_topic_partition_list_destroy(cparts)
cerr := C.rd_kafka_pause_partitions(c.handle.rk, cparts)
if cerr != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return newError(cerr)
}
return nil
}
// Resume consumption for the provided list of partitions
func (c *Consumer) Resume(partitions []TopicPartition) (err error) {
cparts := newCPartsFromTopicPartitions(partitions)
defer C.rd_kafka_topic_partition_list_destroy(cparts)
cerr := C.rd_kafka_resume_partitions(c.handle.rk, cparts)
if cerr != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return newError(cerr)
}
return nil
}
// SetOAuthBearerToken sets the the data to be transmitted
// to a broker during SASL/OAUTHBEARER authentication. It will return nil
// on success, otherwise an error if:
// 1) the token data is invalid (meaning an expiration time in the past
// or either a token value or an extension key or value that does not meet
// the regular expression requirements as per
// https://tools.ietf.org/html/rfc7628#section-3.1);
// 2) SASL/OAUTHBEARER is not supported by the underlying librdkafka build;
// 3) SASL/OAUTHBEARER is supported but is not configured as the client's
// authentication mechanism.
func (c *Consumer) SetOAuthBearerToken(oauthBearerToken OAuthBearerToken) error {
return c.handle.setOAuthBearerToken(oauthBearerToken)
}
// SetOAuthBearerTokenFailure sets the error message describing why token
// retrieval/setting failed; it also schedules a new token refresh event for 10
// seconds later so the attempt may be retried. It will return nil on
// success, otherwise an error if:
// 1) SASL/OAUTHBEARER is not supported by the underlying librdkafka build;
// 2) SASL/OAUTHBEARER is supported but is not configured as the client's
// authentication mechanism.
func (c *Consumer) SetOAuthBearerTokenFailure(errstr string) error {
return c.handle.setOAuthBearerTokenFailure(errstr)
}
// ConsumerGroupMetadata reflects the current consumer group member metadata.
type ConsumerGroupMetadata struct {
serialized []byte
}
// serializeConsumerGroupMetadata converts a C metadata object to its
// binary representation so we don't have to hold on to the C object,
// which would require an explicit .Close().
func serializeConsumerGroupMetadata(cgmd *C.rd_kafka_consumer_group_metadata_t) ([]byte, error) {
var cBuffer *C.void
var cSize C.size_t
cError := C.rd_kafka_consumer_group_metadata_write(cgmd,
(*unsafe.Pointer)(unsafe.Pointer(&cBuffer)), &cSize)
if cError != nil {
return nil, newErrorFromCErrorDestroy(cError)
}
defer C.rd_kafka_mem_free(nil, unsafe.Pointer(cBuffer))
return C.GoBytes(unsafe.Pointer(cBuffer), C.int(cSize)), nil
}
// deserializeConsumerGroupMetadata converts a serialized metadata object
// back to a C object.
func deserializeConsumerGroupMetadata(serialized []byte) (*C.rd_kafka_consumer_group_metadata_t, error) {
var cgmd *C.rd_kafka_consumer_group_metadata_t
cSerialized := C.CBytes(serialized)
defer C.free(cSerialized)
cError := C.rd_kafka_consumer_group_metadata_read(
&cgmd, cSerialized, C.size_t(len(serialized)))
if cError != nil {
return nil, newErrorFromCErrorDestroy(cError)
}
return cgmd, nil
}
// GetConsumerGroupMetadata returns the consumer's current group metadata.
// This object should be passed to the transactional producer's
// SendOffsetsToTransaction() API.
func (c *Consumer) GetConsumerGroupMetadata() (*ConsumerGroupMetadata, error) {
cgmd := C.rd_kafka_consumer_group_metadata(c.handle.rk)
if cgmd == nil {
return nil, NewError(ErrState, "Consumer group metadata not available", false)
}
defer C.rd_kafka_consumer_group_metadata_destroy(cgmd)
serialized, err := serializeConsumerGroupMetadata(cgmd)
if err != nil {
return nil, err
}
return &ConsumerGroupMetadata{serialized}, nil
}
// NewTestConsumerGroupMetadata creates a new consumer group metadata instance
// mainly for testing use.
// Use GetConsumerGroupMetadata() to retrieve the real metadata.
func NewTestConsumerGroupMetadata(groupID string) (*ConsumerGroupMetadata, error) {
cGroupID := C.CString(groupID)
defer C.free(unsafe.Pointer(cGroupID))
cgmd := C.rd_kafka_consumer_group_metadata_new(cGroupID)
if cgmd == nil {
return nil, NewError(ErrInvalidArg, "Failed to create metadata object", false)
}
defer C.rd_kafka_consumer_group_metadata_destroy(cgmd)
serialized, err := serializeConsumerGroupMetadata(cgmd)
if err != nil {
return nil, err
}
return &ConsumerGroupMetadata{serialized}, nil
}
// cEventToRebalanceEvent returns an Event (AssignedPartitions or RevokedPartitions)
// based on the specified rkev.
func cEventToRebalanceEvent(rkev *C.rd_kafka_event_t) Event {
if C.rd_kafka_event_error(rkev) == C.RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS {
var ev AssignedPartitions
ev.Partitions = newTopicPartitionsFromCparts(C.rd_kafka_event_topic_partition_list(rkev))
return ev
} else if C.rd_kafka_event_error(rkev) == C.RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS {
var ev RevokedPartitions
ev.Partitions = newTopicPartitionsFromCparts(C.rd_kafka_event_topic_partition_list(rkev))
return ev
} else {
panic(fmt.Sprintf("Unable to create rebalance event from C type %s",
C.GoString(C.rd_kafka_err2name(C.rd_kafka_event_error(rkev)))))
}
}
// handleRebalanceEvent handles a assign/rebalance rebalance event.
//
// If the app provided a RebalanceCb to Subscribe*() or
// has go.application.rebalance.enable=true we create an event
// and forward it to the application thru the RebalanceCb or the
// Events channel respectively.
// Since librdkafka requires the rebalance event to be "acked" by
// the application (by calling *assign()) to synchronize state we keep track
// of if the application performed *Assign() or *Unassign(), but this only
// works for the non-channel case. For the channel case we assume the
// application calls *Assign() or *Unassign().
// Failure to do so will "hang" the consumer, e.g., it wont start consuming
// and it wont close cleanly, so this error case should be visible
// immediately to the application developer.
//
// In the polling case (not channel based consumer) the rebalance event
// is returned in retval, else nil is returned.
func (c *Consumer) handleRebalanceEvent(channel chan Event, rkev *C.rd_kafka_event_t) (retval Event) {
var ev Event
if c.rebalanceCb != nil || c.appRebalanceEnable {
// Application has a rebalance callback or has enabled
// rebalances on the events channel, create the appropriate Event.
ev = cEventToRebalanceEvent(rkev)
}
if channel != nil && c.appRebalanceEnable && c.rebalanceCb == nil {
// Channel-based consumer with rebalancing enabled,
// return the rebalance event and rely on the application
// to call *Assign() / *Unassign().
return ev
}
// Call the application's rebalance callback, if any.
if c.rebalanceCb != nil {
// Mark .appReassigned as false to keep track of whether the
// application called *Assign() / *Unassign().
c.appReassigned = false
c.rebalanceCb(c, ev)
if c.appReassigned {
// Rebalance event handled by application.
return nil
}
}
// Either there was no rebalance callback, or the application
// did not call *Assign / *Unassign, so we need to do it.
isCooperative := c.GetRebalanceProtocol() == "COOPERATIVE"
var cError *C.rd_kafka_error_t
var cErr C.rd_kafka_resp_err_t
if C.rd_kafka_event_error(rkev) == C.RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS {
// Assign partitions
if isCooperative {
cError = C.rd_kafka_incremental_assign(
c.handle.rk,
C.rd_kafka_event_topic_partition_list(rkev))
} else {
cErr = C.rd_kafka_assign(
c.handle.rk,
C.rd_kafka_event_topic_partition_list(rkev))
}
} else {
// Revoke partitions
if isCooperative {
cError = C.rd_kafka_incremental_unassign(
c.handle.rk,
C.rd_kafka_event_topic_partition_list(rkev))
} else {
cErr = C.rd_kafka_assign(c.handle.rk, nil)
}
}
// If the *assign() call returned error, forward it to the
// the consumer's Events() channel for visibility.
if cError != nil {
c.events <- newErrorFromCErrorDestroy(cError)
} else if cErr != 0 {
c.events <- newError(cErr)
}
return nil
}

31
vendor/github.com/confluentinc/confluent-kafka-go/kafka/context.go generated vendored

@ -0,0 +1,31 @@
/**
* Copyright 2019 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package kafka
import (
"context"
"time"
)
// Timeout returns the remaining time after which work done on behalf of this context should be
// canceled, or ok==false if no deadline/timeout is set.
func timeout(ctx context.Context) (timeout time.Duration, ok bool) {
if deadline, ok := ctx.Deadline(); ok {
return deadline.Sub(time.Now()), true
}
return 0, false
}

154
vendor/github.com/confluentinc/confluent-kafka-go/kafka/error.go generated vendored

@ -0,0 +1,154 @@
package kafka
/**
* Copyright 2016 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Automatically generate error codes from librdkafka
// See README for instructions
//go:generate $GOPATH/bin/go_rdkafka_generr generated_errors.go
/*
#include <stdlib.h>
#include "select_rdkafka.h"
*/
import "C"
import (
"fmt"
"unsafe"
)
// Error provides a Kafka-specific error container
type Error struct {
code ErrorCode
str string
fatal bool
retriable bool
txnRequiresAbort bool
}
func newError(code C.rd_kafka_resp_err_t) (err Error) {
return Error{code: ErrorCode(code)}
}
// NewError creates a new Error.
func NewError(code ErrorCode, str string, fatal bool) (err Error) {
return Error{code: code, str: str, fatal: fatal}
}
func newErrorFromString(code ErrorCode, str string) (err Error) {
return Error{code: code, str: str}
}
func newErrorFromCString(code C.rd_kafka_resp_err_t, cstr *C.char) (err Error) {
var str string
if cstr != nil {
str = C.GoString(cstr)
} else {
str = ""
}
return Error{code: ErrorCode(code), str: str}
}
func newCErrorFromString(code C.rd_kafka_resp_err_t, str string) (err Error) {
return newErrorFromString(ErrorCode(code), str)
}
// newErrorFromCError creates a new Error instance and destroys
// the passed cError.
func newErrorFromCErrorDestroy(cError *C.rd_kafka_error_t) Error {
defer C.rd_kafka_error_destroy(cError)
return Error{
code: ErrorCode(C.rd_kafka_error_code(cError)),
str: C.GoString(C.rd_kafka_error_string(cError)),
fatal: cint2bool(C.rd_kafka_error_is_fatal(cError)),
retriable: cint2bool(C.rd_kafka_error_is_retriable(cError)),
txnRequiresAbort: cint2bool(C.rd_kafka_error_txn_requires_abort(cError)),
}
}
// Error returns a human readable representation of an Error
// Same as Error.String()
func (e Error) Error() string {
return e.String()
}
// String returns a human readable representation of an Error
func (e Error) String() string {
var errstr string
if len(e.str) > 0 {
errstr = e.str
} else {
errstr = e.code.String()
}
if e.IsFatal() {
return fmt.Sprintf("Fatal error: %s", errstr)
}
return errstr
}
// Code returns the ErrorCode of an Error
func (e Error) Code() ErrorCode {
return e.code
}
// IsFatal returns true if the error is a fatal error.
// A fatal error indicates the client instance is no longer operable and
// should be terminated. Typical causes include non-recoverable
// idempotent producer errors.
func (e Error) IsFatal() bool {
return e.fatal
}
// IsRetriable returns true if the operation that caused this error
// may be retried.
// This flag is currently only set by the Transactional producer API.
func (e Error) IsRetriable() bool {
return e.retriable
}
// TxnRequiresAbort returns true if the error is an abortable transaction error
// that requires the application to abort the current transaction with
// AbortTransaction() and start a new transaction with BeginTransaction()
// if it wishes to proceed with transactional operations.
// This flag is only set by the Transactional producer API.
func (e Error) TxnRequiresAbort() bool {
return e.txnRequiresAbort
}
// getFatalError returns an Error object if the client instance has raised a fatal error, else nil.
func getFatalError(H Handle) error {
cErrstr := (*C.char)(C.malloc(C.size_t(512)))
defer C.free(unsafe.Pointer(cErrstr))
cErr := C.rd_kafka_fatal_error(H.gethandle().rk, cErrstr, 512)
if int(cErr) == 0 {
return nil
}
err := newErrorFromCString(cErr, cErrstr)
err.fatal = true
return err
}
// testFatalError triggers a fatal error in the underlying client.
// This is to be used strictly for testing purposes.
func testFatalError(H Handle, code ErrorCode, str string) ErrorCode {
return ErrorCode(C.rd_kafka_test_fatal_error(H.gethandle().rk, C.rd_kafka_resp_err_t(code), C.CString(str)))
}

112
vendor/github.com/confluentinc/confluent-kafka-go/kafka/error_gen.go generated vendored

@ -0,0 +1,112 @@
package kafka
/**
* Copyright 2020 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Automatically generate error codes from librdkafka
// See README for instructions
//go:generate $GOPATH/bin/go_rdkafka_generr generated_errors.go
/*
#include <stdlib.h>
#include "select_rdkafka.h"
static const char *errdesc_to_string (const struct rd_kafka_err_desc *ed, int idx) {
return ed[idx].name;
}
static const char *errdesc_to_desc (const struct rd_kafka_err_desc *ed, int idx) {
return ed[idx].desc;
}
*/
import "C"
import (
"fmt"
"os"
"strings"
"time"
)
// camelCase transforms a snake_case string to camelCase.
func camelCase(s string) string {
ret := ""
for _, v := range strings.Split(s, "_") {
if len(v) == 0 {
continue
}
ret += strings.ToUpper((string)(v[0])) + strings.ToLower(v[1:])
}
return ret
}
// WriteErrorCodes writes Go error code constants to file from the
// librdkafka error codes.
// This function is not intended for public use.
func WriteErrorCodes(f *os.File) {
f.WriteString("package kafka\n")
now := time.Now()
f.WriteString(fmt.Sprintf("// Copyright 2016-%d Confluent Inc.\n", now.Year()))
f.WriteString(fmt.Sprintf("// AUTOMATICALLY GENERATED ON %v USING librdkafka %s\n",
now, C.GoString(C.rd_kafka_version_str())))
var errdescs *C.struct_rd_kafka_err_desc
var csize C.size_t
C.rd_kafka_get_err_descs(&errdescs, &csize)
f.WriteString(`
/*
#include "select_rdkafka.h"
*/
import "C"
// ErrorCode is the integer representation of local and broker error codes
type ErrorCode int
// String returns a human readable representation of an error code
func (c ErrorCode) String() string {
return C.GoString(C.rd_kafka_err2str(C.rd_kafka_resp_err_t(c)))
}
const (
`)
for i := 0; i < int(csize); i++ {
orig := C.GoString(C.errdesc_to_string(errdescs, C.int(i)))
if len(orig) == 0 {
continue
}
desc := C.GoString(C.errdesc_to_desc(errdescs, C.int(i)))
if len(desc) == 0 {
continue
}
errname := "Err" + camelCase(orig)
// Special handling to please golint
// Eof -> EOF
// Id -> ID
errname = strings.Replace(errname, "Eof", "EOF", -1)
errname = strings.Replace(errname, "Id", "ID", -1)
f.WriteString(fmt.Sprintf(" // %s %s\n", errname, desc))
f.WriteString(fmt.Sprintf(" %s ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_%s)\n",
errname, orig))
}
f.WriteString(")\n")
}

316
vendor/github.com/confluentinc/confluent-kafka-go/kafka/event.go generated vendored

@ -0,0 +1,316 @@
package kafka
/**
* Copyright 2016 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import (
"fmt"
"os"
"unsafe"
)
/*
#include <stdlib.h>
#include "select_rdkafka.h"
#include "glue_rdkafka.h"
void chdrs_to_tmphdrs (glue_msg_t *gMsg) {
size_t i = 0;
const char *name;
const void *val;
size_t size;
rd_kafka_headers_t *chdrs;
if (rd_kafka_message_headers(gMsg->msg, &chdrs)) {
gMsg->tmphdrs = NULL;
gMsg->tmphdrsCnt = 0;
return;
}
gMsg->tmphdrsCnt = rd_kafka_header_cnt(chdrs);
gMsg->tmphdrs = malloc(sizeof(*gMsg->tmphdrs) * gMsg->tmphdrsCnt);
while (!rd_kafka_header_get_all(chdrs, i,
&gMsg->tmphdrs[i].key,
&gMsg->tmphdrs[i].val,
(size_t *)&gMsg->tmphdrs[i].size))
i++;
}
rd_kafka_event_t *_rk_queue_poll (rd_kafka_queue_t *rkq, int timeoutMs,
rd_kafka_event_type_t *evtype,
glue_msg_t *gMsg,
rd_kafka_event_t *prev_rkev) {
rd_kafka_event_t *rkev;
if (prev_rkev)
rd_kafka_event_destroy(prev_rkev);
rkev = rd_kafka_queue_poll(rkq, timeoutMs);
*evtype = rd_kafka_event_type(rkev);
if (*evtype == RD_KAFKA_EVENT_FETCH) {
gMsg->msg = (rd_kafka_message_t *)rd_kafka_event_message_next(rkev);
gMsg->ts = rd_kafka_message_timestamp(gMsg->msg, &gMsg->tstype);
if (gMsg->want_hdrs)
chdrs_to_tmphdrs(gMsg);
}
return rkev;
}
*/
import "C"
func chdrsToTmphdrs(gMsg *C.glue_msg_t) {
C.chdrs_to_tmphdrs(gMsg)
}
// Event generic interface
type Event interface {
// String returns a human-readable representation of the event
String() string
}
// Specific event types
// Stats statistics event
type Stats struct {
statsJSON string
}
func (e Stats) String() string {
return e.statsJSON
}
// AssignedPartitions consumer group rebalance event: assigned partition set
type AssignedPartitions struct {
Partitions []TopicPartition
}
func (e AssignedPartitions) String() string {
return fmt.Sprintf("AssignedPartitions: %v", e.Partitions)
}
// RevokedPartitions consumer group rebalance event: revoked partition set
type RevokedPartitions struct {
Partitions []TopicPartition
}
func (e RevokedPartitions) String() string {
return fmt.Sprintf("RevokedPartitions: %v", e.Partitions)
}
// PartitionEOF consumer reached end of partition
// Needs to be explicitly enabled by setting the `enable.partition.eof`
// configuration property to true.
type PartitionEOF TopicPartition
func (p PartitionEOF) String() string {
return fmt.Sprintf("EOF at %s", TopicPartition(p))
}
// OffsetsCommitted reports committed offsets
type OffsetsCommitted struct {
Error error
Offsets []TopicPartition
}
func (o OffsetsCommitted) String() string {
return fmt.Sprintf("OffsetsCommitted (%v, %v)", o.Error, o.Offsets)
}
// OAuthBearerTokenRefresh indicates token refresh is required
type OAuthBearerTokenRefresh struct {
// Config is the value of the sasl.oauthbearer.config property
Config string
}
func (o OAuthBearerTokenRefresh) String() string {
return "OAuthBearerTokenRefresh"
}
// eventPoll polls an event from the handler's C rd_kafka_queue_t,
// translates it into an Event type and then sends on `channel` if non-nil, else returns the Event.
// term_chan is an optional channel to monitor along with producing to channel
// to indicate that `channel` is being terminated.
// returns (event Event, terminate Bool) tuple, where Terminate indicates
// if termChan received a termination event.
func (h *handle) eventPoll(channel chan Event, timeoutMs int, maxEvents int, termChan chan bool) (Event, bool) {
var prevRkev *C.rd_kafka_event_t
term := false
var retval Event
if channel == nil {
maxEvents = 1
}
out:
for evcnt := 0; evcnt < maxEvents; evcnt++ {
var evtype C.rd_kafka_event_type_t
var gMsg C.glue_msg_t
gMsg.want_hdrs = C.int8_t(bool2cint(h.msgFields.Headers))
rkev := C._rk_queue_poll(h.rkq, C.int(timeoutMs), &evtype, &gMsg, prevRkev)
prevRkev = rkev
timeoutMs = 0
retval = nil
switch evtype {
case C.RD_KAFKA_EVENT_FETCH:
// Consumer fetch event, new message.
// Extracted into temporary gMsg for optimization
retval = h.newMessageFromGlueMsg(&gMsg)
case C.RD_KAFKA_EVENT_REBALANCE:
// Consumer rebalance event
retval = h.c.handleRebalanceEvent(channel, rkev)
case C.RD_KAFKA_EVENT_ERROR:
// Error event
cErr := C.rd_kafka_event_error(rkev)
if cErr == C.RD_KAFKA_RESP_ERR__PARTITION_EOF {
crktpar := C.rd_kafka_event_topic_partition(rkev)
if crktpar == nil {
break
}
defer C.rd_kafka_topic_partition_destroy(crktpar)
var peof PartitionEOF
setupTopicPartitionFromCrktpar((*TopicPartition)(&peof), crktpar)
retval = peof
} else if int(C.rd_kafka_event_error_is_fatal(rkev)) != 0 {
// A fatal error has been raised.
// Extract the actual error from the client
// instance and return a new Error with
// fatal set to true.
cFatalErrstrSize := C.size_t(512)
cFatalErrstr := (*C.char)(C.malloc(cFatalErrstrSize))
defer C.free(unsafe.Pointer(cFatalErrstr))
cFatalErr := C.rd_kafka_fatal_error(h.rk, cFatalErrstr, cFatalErrstrSize)
fatalErr := newErrorFromCString(cFatalErr, cFatalErrstr)
fatalErr.fatal = true
retval = fatalErr
} else {
retval = newErrorFromCString(cErr, C.rd_kafka_event_error_string(rkev))
}
case C.RD_KAFKA_EVENT_STATS:
retval = &Stats{C.GoString(C.rd_kafka_event_stats(rkev))}
case C.RD_KAFKA_EVENT_DR:
// Producer Delivery Report event
// Each such event contains delivery reports for all
// messages in the produced batch.
// Forward delivery reports to per-message's response channel
// or to the global Producer.Events channel, or none.
rkmessages := make([]*C.rd_kafka_message_t, int(C.rd_kafka_event_message_count(rkev)))
cnt := int(C.rd_kafka_event_message_array(rkev, (**C.rd_kafka_message_t)(unsafe.Pointer(&rkmessages[0])), C.size_t(len(rkmessages))))
for _, rkmessage := range rkmessages[:cnt] {
msg := h.newMessageFromC(rkmessage)
var ch *chan Event
if rkmessage._private != nil {
// Find cgoif by id
cg, found := h.cgoGet((int)((uintptr)(rkmessage._private)))
if found {
cdr := cg.(cgoDr)
if cdr.deliveryChan != nil {
ch = &cdr.deliveryChan
}
msg.Opaque = cdr.opaque
}
}
if ch == nil && h.fwdDr {
ch = &channel
}
if ch != nil {
select {
case *ch <- msg:
case <-termChan:
retval = nil
term = true
break out
}
} else {
retval = msg
break out
}
}
case C.RD_KAFKA_EVENT_OFFSET_COMMIT:
// Offsets committed
cErr := C.rd_kafka_event_error(rkev)
coffsets := C.rd_kafka_event_topic_partition_list(rkev)
var offsets []TopicPartition
if coffsets != nil {
offsets = newTopicPartitionsFromCparts(coffsets)
}
if cErr != C.RD_KAFKA_RESP_ERR_NO_ERROR {
retval = OffsetsCommitted{newErrorFromCString(cErr, C.rd_kafka_event_error_string(rkev)), offsets}
} else {
retval = OffsetsCommitted{nil, offsets}
}
case C.RD_KAFKA_EVENT_OAUTHBEARER_TOKEN_REFRESH:
ev := OAuthBearerTokenRefresh{C.GoString(C.rd_kafka_event_config_string(rkev))}
retval = ev
case C.RD_KAFKA_EVENT_NONE:
// poll timed out: no events available
break out
default:
if rkev != nil {
fmt.Fprintf(os.Stderr, "Ignored event %s\n",
C.GoString(C.rd_kafka_event_name(rkev)))
}
}
if retval != nil {
if channel != nil {
select {
case channel <- retval:
case <-termChan:
retval = nil
term = true
break out
}
} else {
break out
}
}
}
if prevRkev != nil {
C.rd_kafka_event_destroy(prevRkev)
}
return retval, term
}

337
vendor/github.com/confluentinc/confluent-kafka-go/kafka/generated_errors.go generated vendored

@ -0,0 +1,337 @@
package kafka
// Copyright 2016-2021 Confluent Inc.
// AUTOMATICALLY GENERATED ON 2021-12-08 12:44:39.243338672 +0100 CET m=+0.000248284 USING librdkafka 1.8.2
/*
#include "select_rdkafka.h"
*/
import "C"
// ErrorCode is the integer representation of local and broker error codes
type ErrorCode int
// String returns a human readable representation of an error code
func (c ErrorCode) String() string {
return C.GoString(C.rd_kafka_err2str(C.rd_kafka_resp_err_t(c)))
}
const (
// ErrBadMsg Local: Bad message format
ErrBadMsg ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__BAD_MSG)
// ErrBadCompression Local: Invalid compressed data
ErrBadCompression ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__BAD_COMPRESSION)
// ErrDestroy Local: Broker handle destroyed
ErrDestroy ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__DESTROY)
// ErrFail Local: Communication failure with broker
ErrFail ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__FAIL)
// ErrTransport Local: Broker transport failure
ErrTransport ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__TRANSPORT)
// ErrCritSysResource Local: Critical system resource failure
ErrCritSysResource ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__CRIT_SYS_RESOURCE)
// ErrResolve Local: Host resolution failure
ErrResolve ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__RESOLVE)
// ErrMsgTimedOut Local: Message timed out
ErrMsgTimedOut ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__MSG_TIMED_OUT)
// ErrPartitionEOF Broker: No more messages
ErrPartitionEOF ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__PARTITION_EOF)
// ErrUnknownPartition Local: Unknown partition
ErrUnknownPartition ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION)
// ErrFs Local: File or filesystem error
ErrFs ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__FS)
// ErrUnknownTopic Local: Unknown topic
ErrUnknownTopic ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC)
// ErrAllBrokersDown Local: All broker connections are down
ErrAllBrokersDown ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN)
// ErrInvalidArg Local: Invalid argument or configuration
ErrInvalidArg ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__INVALID_ARG)
// ErrTimedOut Local: Timed out
ErrTimedOut ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__TIMED_OUT)
// ErrQueueFull Local: Queue full
ErrQueueFull ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__QUEUE_FULL)
// ErrIsrInsuff Local: ISR count insufficient
ErrIsrInsuff ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__ISR_INSUFF)
// ErrNodeUpdate Local: Broker node update
ErrNodeUpdate ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__NODE_UPDATE)
// ErrSsl Local: SSL error
ErrSsl ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__SSL)
// ErrWaitCoord Local: Waiting for coordinator
ErrWaitCoord ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__WAIT_COORD)
// ErrUnknownGroup Local: Unknown group
ErrUnknownGroup ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__UNKNOWN_GROUP)
// ErrInProgress Local: Operation in progress
ErrInProgress ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__IN_PROGRESS)
// ErrPrevInProgress Local: Previous operation in progress
ErrPrevInProgress ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__PREV_IN_PROGRESS)
// ErrExistingSubscription Local: Existing subscription
ErrExistingSubscription ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__EXISTING_SUBSCRIPTION)
// ErrAssignPartitions Local: Assign partitions
ErrAssignPartitions ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS)
// ErrRevokePartitions Local: Revoke partitions
ErrRevokePartitions ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS)
// ErrConflict Local: Conflicting use
ErrConflict ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__CONFLICT)
// ErrState Local: Erroneous state
ErrState ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__STATE)
// ErrUnknownProtocol Local: Unknown protocol
ErrUnknownProtocol ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__UNKNOWN_PROTOCOL)
// ErrNotImplemented Local: Not implemented
ErrNotImplemented ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED)
// ErrAuthentication Local: Authentication failure
ErrAuthentication ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__AUTHENTICATION)
// ErrNoOffset Local: No offset stored
ErrNoOffset ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__NO_OFFSET)
// ErrOutdated Local: Outdated
ErrOutdated ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__OUTDATED)
// ErrTimedOutQueue Local: Timed out in queue
ErrTimedOutQueue ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__TIMED_OUT_QUEUE)
// ErrUnsupportedFeature Local: Required feature not supported by broker
ErrUnsupportedFeature ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__UNSUPPORTED_FEATURE)
// ErrWaitCache Local: Awaiting cache update
ErrWaitCache ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__WAIT_CACHE)
// ErrIntr Local: Operation interrupted
ErrIntr ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__INTR)
// ErrKeySerialization Local: Key serialization error
ErrKeySerialization ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__KEY_SERIALIZATION)
// ErrValueSerialization Local: Value serialization error
ErrValueSerialization ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__VALUE_SERIALIZATION)
// ErrKeyDeserialization Local: Key deserialization error
ErrKeyDeserialization ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__KEY_DESERIALIZATION)
// ErrValueDeserialization Local: Value deserialization error
ErrValueDeserialization ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__VALUE_DESERIALIZATION)
// ErrPartial Local: Partial response
ErrPartial ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__PARTIAL)
// ErrReadOnly Local: Read-only object
ErrReadOnly ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__READ_ONLY)
// ErrNoent Local: No such entry
ErrNoent ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__NOENT)
// ErrUnderflow Local: Read underflow
ErrUnderflow ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__UNDERFLOW)
// ErrInvalidType Local: Invalid type
ErrInvalidType ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__INVALID_TYPE)
// ErrRetry Local: Retry operation
ErrRetry ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__RETRY)
// ErrPurgeQueue Local: Purged in queue
ErrPurgeQueue ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__PURGE_QUEUE)
// ErrPurgeInflight Local: Purged in flight
ErrPurgeInflight ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__PURGE_INFLIGHT)
// ErrFatal Local: Fatal error
ErrFatal ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__FATAL)
// ErrInconsistent Local: Inconsistent state
ErrInconsistent ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__INCONSISTENT)
// ErrGaplessGuarantee Local: Gap-less ordering would not be guaranteed if proceeding
ErrGaplessGuarantee ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__GAPLESS_GUARANTEE)
// ErrMaxPollExceeded Local: Maximum application poll interval (max.poll.interval.ms) exceeded
ErrMaxPollExceeded ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__MAX_POLL_EXCEEDED)
// ErrUnknownBroker Local: Unknown broker
ErrUnknownBroker ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__UNKNOWN_BROKER)
// ErrNotConfigured Local: Functionality not configured
ErrNotConfigured ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__NOT_CONFIGURED)
// ErrFenced Local: This instance has been fenced by a newer instance
ErrFenced ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__FENCED)
// ErrApplication Local: Application generated error
ErrApplication ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__APPLICATION)
// ErrAssignmentLost Local: Group partition assignment lost
ErrAssignmentLost ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__ASSIGNMENT_LOST)
// ErrNoop Local: No operation performed
ErrNoop ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__NOOP)
// ErrAutoOffsetReset Local: No offset to automatically reset to
ErrAutoOffsetReset ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR__AUTO_OFFSET_RESET)
// ErrUnknown Unknown broker error
ErrUnknown ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_UNKNOWN)
// ErrNoError Success
ErrNoError ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_NO_ERROR)
// ErrOffsetOutOfRange Broker: Offset out of range
ErrOffsetOutOfRange ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_OFFSET_OUT_OF_RANGE)
// ErrInvalidMsg Broker: Invalid message
ErrInvalidMsg ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_MSG)
// ErrUnknownTopicOrPart Broker: Unknown topic or partition
ErrUnknownTopicOrPart ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART)
// ErrInvalidMsgSize Broker: Invalid message size
ErrInvalidMsgSize ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_MSG_SIZE)
// ErrLeaderNotAvailable Broker: Leader not available
ErrLeaderNotAvailable ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE)
// ErrNotLeaderForPartition Broker: Not leader for partition
ErrNotLeaderForPartition ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_NOT_LEADER_FOR_PARTITION)
// ErrRequestTimedOut Broker: Request timed out
ErrRequestTimedOut ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT)
// ErrBrokerNotAvailable Broker: Broker not available
ErrBrokerNotAvailable ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_BROKER_NOT_AVAILABLE)
// ErrReplicaNotAvailable Broker: Replica not available
ErrReplicaNotAvailable ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE)
// ErrMsgSizeTooLarge Broker: Message size too large
ErrMsgSizeTooLarge ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE)
// ErrStaleCtrlEpoch Broker: StaleControllerEpochCode
ErrStaleCtrlEpoch ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_STALE_CTRL_EPOCH)
// ErrOffsetMetadataTooLarge Broker: Offset metadata string too large
ErrOffsetMetadataTooLarge ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_OFFSET_METADATA_TOO_LARGE)
// ErrNetworkException Broker: Broker disconnected before response received
ErrNetworkException ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_NETWORK_EXCEPTION)
// ErrCoordinatorLoadInProgress Broker: Coordinator load in progress
ErrCoordinatorLoadInProgress ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_COORDINATOR_LOAD_IN_PROGRESS)
// ErrCoordinatorNotAvailable Broker: Coordinator not available
ErrCoordinatorNotAvailable ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_COORDINATOR_NOT_AVAILABLE)
// ErrNotCoordinator Broker: Not coordinator
ErrNotCoordinator ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_NOT_COORDINATOR)
// ErrTopicException Broker: Invalid topic
ErrTopicException ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION)
// ErrRecordListTooLarge Broker: Message batch larger than configured server segment size
ErrRecordListTooLarge ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_RECORD_LIST_TOO_LARGE)
// ErrNotEnoughReplicas Broker: Not enough in-sync replicas
ErrNotEnoughReplicas ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_NOT_ENOUGH_REPLICAS)
// ErrNotEnoughReplicasAfterAppend Broker: Message(s) written to insufficient number of in-sync replicas
ErrNotEnoughReplicasAfterAppend ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_NOT_ENOUGH_REPLICAS_AFTER_APPEND)
// ErrInvalidRequiredAcks Broker: Invalid required acks value
ErrInvalidRequiredAcks ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_REQUIRED_ACKS)
// ErrIllegalGeneration Broker: Specified group generation id is not valid
ErrIllegalGeneration ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_ILLEGAL_GENERATION)
// ErrInconsistentGroupProtocol Broker: Inconsistent group protocol
ErrInconsistentGroupProtocol ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INCONSISTENT_GROUP_PROTOCOL)
// ErrInvalidGroupID Broker: Invalid group.id
ErrInvalidGroupID ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_GROUP_ID)
// ErrUnknownMemberID Broker: Unknown member
ErrUnknownMemberID ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_UNKNOWN_MEMBER_ID)
// ErrInvalidSessionTimeout Broker: Invalid session timeout
ErrInvalidSessionTimeout ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_SESSION_TIMEOUT)
// ErrRebalanceInProgress Broker: Group rebalance in progress
ErrRebalanceInProgress ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_REBALANCE_IN_PROGRESS)
// ErrInvalidCommitOffsetSize Broker: Commit offset data size is not valid
ErrInvalidCommitOffsetSize ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_COMMIT_OFFSET_SIZE)
// ErrTopicAuthorizationFailed Broker: Topic authorization failed
ErrTopicAuthorizationFailed ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED)
// ErrGroupAuthorizationFailed Broker: Group authorization failed
ErrGroupAuthorizationFailed ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_GROUP_AUTHORIZATION_FAILED)
// ErrClusterAuthorizationFailed Broker: Cluster authorization failed
ErrClusterAuthorizationFailed ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_CLUSTER_AUTHORIZATION_FAILED)
// ErrInvalidTimestamp Broker: Invalid timestamp
ErrInvalidTimestamp ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_TIMESTAMP)
// ErrUnsupportedSaslMechanism Broker: Unsupported SASL mechanism
ErrUnsupportedSaslMechanism ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_UNSUPPORTED_SASL_MECHANISM)
// ErrIllegalSaslState Broker: Request not valid in current SASL state
ErrIllegalSaslState ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_ILLEGAL_SASL_STATE)
// ErrUnsupportedVersion Broker: API version not supported
ErrUnsupportedVersion ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_UNSUPPORTED_VERSION)
// ErrTopicAlreadyExists Broker: Topic already exists
ErrTopicAlreadyExists ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_TOPIC_ALREADY_EXISTS)
// ErrInvalidPartitions Broker: Invalid number of partitions
ErrInvalidPartitions ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_PARTITIONS)
// ErrInvalidReplicationFactor Broker: Invalid replication factor
ErrInvalidReplicationFactor ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_REPLICATION_FACTOR)
// ErrInvalidReplicaAssignment Broker: Invalid replica assignment
ErrInvalidReplicaAssignment ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_REPLICA_ASSIGNMENT)
// ErrInvalidConfig Broker: Configuration is invalid
ErrInvalidConfig ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_CONFIG)
// ErrNotController Broker: Not controller for cluster
ErrNotController ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_NOT_CONTROLLER)
// ErrInvalidRequest Broker: Invalid request
ErrInvalidRequest ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_REQUEST)
// ErrUnsupportedForMessageFormat Broker: Message format on broker does not support request
ErrUnsupportedForMessageFormat ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_UNSUPPORTED_FOR_MESSAGE_FORMAT)
// ErrPolicyViolation Broker: Policy violation
ErrPolicyViolation ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_POLICY_VIOLATION)
// ErrOutOfOrderSequenceNumber Broker: Broker received an out of order sequence number
ErrOutOfOrderSequenceNumber ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_OUT_OF_ORDER_SEQUENCE_NUMBER)
// ErrDuplicateSequenceNumber Broker: Broker received a duplicate sequence number
ErrDuplicateSequenceNumber ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_DUPLICATE_SEQUENCE_NUMBER)
// ErrInvalidProducerEpoch Broker: Producer attempted an operation with an old epoch
ErrInvalidProducerEpoch ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_PRODUCER_EPOCH)
// ErrInvalidTxnState Broker: Producer attempted a transactional operation in an invalid state
ErrInvalidTxnState ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_TXN_STATE)
// ErrInvalidProducerIDMapping Broker: Producer attempted to use a producer id which is not currently assigned to its transactional id
ErrInvalidProducerIDMapping ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_PRODUCER_ID_MAPPING)
// ErrInvalidTransactionTimeout Broker: Transaction timeout is larger than the maximum value allowed by the broker's max.transaction.timeout.ms
ErrInvalidTransactionTimeout ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_TRANSACTION_TIMEOUT)
// ErrConcurrentTransactions Broker: Producer attempted to update a transaction while another concurrent operation on the same transaction was ongoing
ErrConcurrentTransactions ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_CONCURRENT_TRANSACTIONS)
// ErrTransactionCoordinatorFenced Broker: Indicates that the transaction coordinator sending a WriteTxnMarker is no longer the current coordinator for a given producer
ErrTransactionCoordinatorFenced ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_TRANSACTION_COORDINATOR_FENCED)
// ErrTransactionalIDAuthorizationFailed Broker: Transactional Id authorization failed
ErrTransactionalIDAuthorizationFailed ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_TRANSACTIONAL_ID_AUTHORIZATION_FAILED)
// ErrSecurityDisabled Broker: Security features are disabled
ErrSecurityDisabled ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_SECURITY_DISABLED)
// ErrOperationNotAttempted Broker: Operation not attempted
ErrOperationNotAttempted ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_OPERATION_NOT_ATTEMPTED)
// ErrKafkaStorageError Broker: Disk error when trying to access log file on disk
ErrKafkaStorageError ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_KAFKA_STORAGE_ERROR)
// ErrLogDirNotFound Broker: The user-specified log directory is not found in the broker config
ErrLogDirNotFound ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_LOG_DIR_NOT_FOUND)
// ErrSaslAuthenticationFailed Broker: SASL Authentication failed
ErrSaslAuthenticationFailed ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_SASL_AUTHENTICATION_FAILED)
// ErrUnknownProducerID Broker: Unknown Producer Id
ErrUnknownProducerID ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_UNKNOWN_PRODUCER_ID)
// ErrReassignmentInProgress Broker: Partition reassignment is in progress
ErrReassignmentInProgress ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_REASSIGNMENT_IN_PROGRESS)
// ErrDelegationTokenAuthDisabled Broker: Delegation Token feature is not enabled
ErrDelegationTokenAuthDisabled ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_DELEGATION_TOKEN_AUTH_DISABLED)
// ErrDelegationTokenNotFound Broker: Delegation Token is not found on server
ErrDelegationTokenNotFound ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_DELEGATION_TOKEN_NOT_FOUND)
// ErrDelegationTokenOwnerMismatch Broker: Specified Principal is not valid Owner/Renewer
ErrDelegationTokenOwnerMismatch ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_DELEGATION_TOKEN_OWNER_MISMATCH)
// ErrDelegationTokenRequestNotAllowed Broker: Delegation Token requests are not allowed on this connection
ErrDelegationTokenRequestNotAllowed ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_DELEGATION_TOKEN_REQUEST_NOT_ALLOWED)
// ErrDelegationTokenAuthorizationFailed Broker: Delegation Token authorization failed
ErrDelegationTokenAuthorizationFailed ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_DELEGATION_TOKEN_AUTHORIZATION_FAILED)
// ErrDelegationTokenExpired Broker: Delegation Token is expired
ErrDelegationTokenExpired ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_DELEGATION_TOKEN_EXPIRED)
// ErrInvalidPrincipalType Broker: Supplied principalType is not supported
ErrInvalidPrincipalType ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_PRINCIPAL_TYPE)
// ErrNonEmptyGroup Broker: The group is not empty
ErrNonEmptyGroup ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_NON_EMPTY_GROUP)
// ErrGroupIDNotFound Broker: The group id does not exist
ErrGroupIDNotFound ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_GROUP_ID_NOT_FOUND)
// ErrFetchSessionIDNotFound Broker: The fetch session ID was not found
ErrFetchSessionIDNotFound ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_FETCH_SESSION_ID_NOT_FOUND)
// ErrInvalidFetchSessionEpoch Broker: The fetch session epoch is invalid
ErrInvalidFetchSessionEpoch ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_FETCH_SESSION_EPOCH)
// ErrListenerNotFound Broker: No matching listener
ErrListenerNotFound ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_LISTENER_NOT_FOUND)
// ErrTopicDeletionDisabled Broker: Topic deletion is disabled
ErrTopicDeletionDisabled ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_TOPIC_DELETION_DISABLED)
// ErrFencedLeaderEpoch Broker: Leader epoch is older than broker epoch
ErrFencedLeaderEpoch ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_FENCED_LEADER_EPOCH)
// ErrUnknownLeaderEpoch Broker: Leader epoch is newer than broker epoch
ErrUnknownLeaderEpoch ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_UNKNOWN_LEADER_EPOCH)
// ErrUnsupportedCompressionType Broker: Unsupported compression type
ErrUnsupportedCompressionType ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_UNSUPPORTED_COMPRESSION_TYPE)
// ErrStaleBrokerEpoch Broker: Broker epoch has changed
ErrStaleBrokerEpoch ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_STALE_BROKER_EPOCH)
// ErrOffsetNotAvailable Broker: Leader high watermark is not caught up
ErrOffsetNotAvailable ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_OFFSET_NOT_AVAILABLE)
// ErrMemberIDRequired Broker: Group member needs a valid member ID
ErrMemberIDRequired ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_MEMBER_ID_REQUIRED)
// ErrPreferredLeaderNotAvailable Broker: Preferred leader was not available
ErrPreferredLeaderNotAvailable ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_PREFERRED_LEADER_NOT_AVAILABLE)
// ErrGroupMaxSizeReached Broker: Consumer group has reached maximum size
ErrGroupMaxSizeReached ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_GROUP_MAX_SIZE_REACHED)
// ErrFencedInstanceID Broker: Static consumer fenced by other consumer with same group.instance.id
ErrFencedInstanceID ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_FENCED_INSTANCE_ID)
// ErrEligibleLeadersNotAvailable Broker: Eligible partition leaders are not available
ErrEligibleLeadersNotAvailable ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_ELIGIBLE_LEADERS_NOT_AVAILABLE)
// ErrElectionNotNeeded Broker: Leader election not needed for topic partition
ErrElectionNotNeeded ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_ELECTION_NOT_NEEDED)
// ErrNoReassignmentInProgress Broker: No partition reassignment is in progress
ErrNoReassignmentInProgress ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_NO_REASSIGNMENT_IN_PROGRESS)
// ErrGroupSubscribedToTopic Broker: Deleting offsets of a topic while the consumer group is subscribed to it
ErrGroupSubscribedToTopic ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_GROUP_SUBSCRIBED_TO_TOPIC)
// ErrInvalidRecord Broker: Broker failed to validate record
ErrInvalidRecord ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_RECORD)
// ErrUnstableOffsetCommit Broker: There are unstable offsets that need to be cleared
ErrUnstableOffsetCommit ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_UNSTABLE_OFFSET_COMMIT)
// ErrThrottlingQuotaExceeded Broker: Throttling quota has been exceeded
ErrThrottlingQuotaExceeded ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_THROTTLING_QUOTA_EXCEEDED)
// ErrProducerFenced Broker: There is a newer producer with the same transactionalId which fences the current one
ErrProducerFenced ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_PRODUCER_FENCED)
// ErrResourceNotFound Broker: Request illegally referred to resource that does not exist
ErrResourceNotFound ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_RESOURCE_NOT_FOUND)
// ErrDuplicateResource Broker: Request illegally referred to the same resource twice
ErrDuplicateResource ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_DUPLICATE_RESOURCE)
// ErrUnacceptableCredential Broker: Requested credential would not meet criteria for acceptability
ErrUnacceptableCredential ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_UNACCEPTABLE_CREDENTIAL)
// ErrInconsistentVoterSet Broker: Indicates that the either the sender or recipient of a voter-only request is not one of the expected voters
ErrInconsistentVoterSet ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INCONSISTENT_VOTER_SET)
// ErrInvalidUpdateVersion Broker: Invalid update version
ErrInvalidUpdateVersion ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_INVALID_UPDATE_VERSION)
// ErrFeatureUpdateFailed Broker: Unable to update finalized features due to server error
ErrFeatureUpdateFailed ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_FEATURE_UPDATE_FAILED)
// ErrPrincipalDeserializationFailure Broker: Request principal deserialization failed during forwarding
ErrPrincipalDeserializationFailure ErrorCode = ErrorCode(C.RD_KAFKA_RESP_ERR_PRINCIPAL_DESERIALIZATION_FAILURE)
)

48
vendor/github.com/confluentinc/confluent-kafka-go/kafka/glue_rdkafka.h generated vendored

@ -0,0 +1,48 @@
/**
* Copyright 2016 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
/**
* Glue between Go, Cgo and librdkafka
*/
/**
* Temporary C to Go header representation
*/
typedef struct tmphdr_s {
const char *key;
const void *val; // producer: malloc()ed by Go code if size > 0
// consumer: owned by librdkafka
ssize_t size;
} tmphdr_t;
/**
* @struct This is a glue struct used by the C code in this client to
* effectively map fields from a librdkafka rd_kafka_message_t
* to something usable in Go with as few CGo calls as possible.
*/
typedef struct glue_msg_s {
rd_kafka_message_t *msg;
rd_kafka_timestamp_type_t tstype;
int64_t ts;
tmphdr_t *tmphdrs;
size_t tmphdrsCnt;
int8_t want_hdrs; /**< If true, copy headers */
} glue_msg_t;

379
vendor/github.com/confluentinc/confluent-kafka-go/kafka/handle.go generated vendored

@ -0,0 +1,379 @@
package kafka
/**
* Copyright 2016 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import (
"fmt"
"strings"
"sync"
"time"
"unsafe"
)
/*
#include "select_rdkafka.h"
#include <stdlib.h>
*/
import "C"
// OAuthBearerToken represents the data to be transmitted
// to a broker during SASL/OAUTHBEARER authentication.
type OAuthBearerToken struct {
// Token value, often (but not necessarily) a JWS compact serialization
// as per https://tools.ietf.org/html/rfc7515#section-3.1; it must meet
// the regular expression for a SASL/OAUTHBEARER value defined at
// https://tools.ietf.org/html/rfc7628#section-3.1
TokenValue string
// Metadata about the token indicating when it expires (local time);
// it must represent a time in the future
Expiration time.Time
// Metadata about the token indicating the Kafka principal name
// to which it applies (for example, "admin")
Principal string
// SASL extensions, if any, to be communicated to the broker during
// authentication (all keys and values of which must meet the regular
// expressions defined at https://tools.ietf.org/html/rfc7628#section-3.1,
// and it must not contain the reserved "auth" key)
Extensions map[string]string
}
// Handle represents a generic client handle containing common parts for
// both Producer and Consumer.
type Handle interface {
// SetOAuthBearerToken sets the the data to be transmitted
// to a broker during SASL/OAUTHBEARER authentication. It will return nil
// on success, otherwise an error if:
// 1) the token data is invalid (meaning an expiration time in the past
// or either a token value or an extension key or value that does not meet
// the regular expression requirements as per
// https://tools.ietf.org/html/rfc7628#section-3.1);
// 2) SASL/OAUTHBEARER is not supported by the underlying librdkafka build;
// 3) SASL/OAUTHBEARER is supported but is not configured as the client's
// authentication mechanism.
SetOAuthBearerToken(oauthBearerToken OAuthBearerToken) error
// SetOAuthBearerTokenFailure sets the error message describing why token
// retrieval/setting failed; it also schedules a new token refresh event for 10
// seconds later so the attempt may be retried. It will return nil on
// success, otherwise an error if:
// 1) SASL/OAUTHBEARER is not supported by the underlying librdkafka build;
// 2) SASL/OAUTHBEARER is supported but is not configured as the client's
// authentication mechanism.
SetOAuthBearerTokenFailure(errstr string) error
// gethandle() returns the internal handle struct pointer
gethandle() *handle
}
// Common instance handle for both Producer and Consumer
type handle struct {
rk *C.rd_kafka_t
rkq *C.rd_kafka_queue_t
// Forward logs from librdkafka log queue to logs channel.
logs chan LogEvent
logq *C.rd_kafka_queue_t
closeLogsChan bool
// Topic <-> rkt caches
rktCacheLock sync.Mutex
// topic name -> rkt cache
rktCache map[string]*C.rd_kafka_topic_t
// rkt -> topic name cache
rktNameCache map[*C.rd_kafka_topic_t]string
// Cached instance name to avoid CGo call in String()
name string
//
// cgo map
// Maps C callbacks based on cgoid back to its Go object
cgoLock sync.Mutex
cgoidNext uintptr
cgomap map[int]cgoif
//
// producer
//
p *Producer
// Forward delivery reports on Producer.Events channel
fwdDr bool
// Enabled message fields for delivery reports and consumed messages.
msgFields *messageFields
//
// consumer
//
c *Consumer
// WaitGroup to wait for spawned go-routines to finish.
waitGroup sync.WaitGroup
}
func (h *handle) String() string {
return h.name
}
func (h *handle) setup() {
h.rktCache = make(map[string]*C.rd_kafka_topic_t)
h.rktNameCache = make(map[*C.rd_kafka_topic_t]string)
h.cgomap = make(map[int]cgoif)
h.name = C.GoString(C.rd_kafka_name(h.rk))
if h.msgFields == nil {
h.msgFields = newMessageFields()
}
}
func (h *handle) cleanup() {
if h.logs != nil {
C.rd_kafka_queue_destroy(h.logq)
if h.closeLogsChan {
close(h.logs)
}
}
for _, crkt := range h.rktCache {
C.rd_kafka_topic_destroy(crkt)
}
if h.rkq != nil {
C.rd_kafka_queue_destroy(h.rkq)
}
}
func (h *handle) setupLogQueue(logsChan chan LogEvent, termChan chan bool) {
if logsChan == nil {
logsChan = make(chan LogEvent, 10000)
h.closeLogsChan = true
}
h.logs = logsChan
// Let librdkafka forward logs to our log queue instead of the main queue
h.logq = C.rd_kafka_queue_new(h.rk)
C.rd_kafka_set_log_queue(h.rk, h.logq)
// Start a polling goroutine to consume the log queue
h.waitGroup.Add(1)
go func() {
h.pollLogEvents(h.logs, 100, termChan)
h.waitGroup.Done()
}()
}
// getRkt0 finds or creates and returns a C topic_t object from the local cache.
func (h *handle) getRkt0(topic string, ctopic *C.char, doLock bool) (crkt *C.rd_kafka_topic_t) {
if doLock {
h.rktCacheLock.Lock()
defer h.rktCacheLock.Unlock()
}
crkt, ok := h.rktCache[topic]
if ok {
return crkt
}
if ctopic == nil {
ctopic = C.CString(topic)
defer C.free(unsafe.Pointer(ctopic))
}
crkt = C.rd_kafka_topic_new(h.rk, ctopic, nil)
if crkt == nil {
panic(fmt.Sprintf("Unable to create new C topic \"%s\": %s",
topic, C.GoString(C.rd_kafka_err2str(C.rd_kafka_last_error()))))
}
h.rktCache[topic] = crkt
h.rktNameCache[crkt] = topic
return crkt
}
// getRkt finds or creates and returns a C topic_t object from the local cache.
func (h *handle) getRkt(topic string) (crkt *C.rd_kafka_topic_t) {
return h.getRkt0(topic, nil, true)
}
// getTopicNameFromRkt returns the topic name for a C topic_t object, preferably
// using the local cache to avoid a cgo call.
func (h *handle) getTopicNameFromRkt(crkt *C.rd_kafka_topic_t) (topic string) {
h.rktCacheLock.Lock()
defer h.rktCacheLock.Unlock()
topic, ok := h.rktNameCache[crkt]
if ok {
return topic
}
// we need our own copy/refcount of the crkt
ctopic := C.rd_kafka_topic_name(crkt)
topic = C.GoString(ctopic)
crkt = h.getRkt0(topic, ctopic, false /* dont lock */)
return topic
}
// cgoif is a generic interface for holding Go state passed as opaque
// value to the C code.
// Since pointers to complex Go types cannot be passed to C we instead create
// a cgoif object, generate a unique id that is added to the cgomap,
// and then pass that id to the C code. When the C code callback is called we
// use the id to look up the cgoif object in the cgomap.
type cgoif interface{}
// delivery report cgoif container
type cgoDr struct {
deliveryChan chan Event
opaque interface{}
}
// cgoPut adds object cg to the handle's cgo map and returns a
// unique id for the added entry.
// Thread-safe.
// FIXME: the uniquity of the id is questionable over time.
func (h *handle) cgoPut(cg cgoif) (cgoid int) {
h.cgoLock.Lock()
defer h.cgoLock.Unlock()
h.cgoidNext++
if h.cgoidNext == 0 {
h.cgoidNext++
}
cgoid = (int)(h.cgoidNext)
h.cgomap[cgoid] = cg
return cgoid
}
// cgoGet looks up cgoid in the cgo map, deletes the reference from the map
// and returns the object, if found. Else returns nil, false.
// Thread-safe.
func (h *handle) cgoGet(cgoid int) (cg cgoif, found bool) {
if cgoid == 0 {
return nil, false
}
h.cgoLock.Lock()
defer h.cgoLock.Unlock()
cg, found = h.cgomap[cgoid]
if found {
delete(h.cgomap, cgoid)
}
return cg, found
}
// setOauthBearerToken - see rd_kafka_oauthbearer_set_token()
func (h *handle) setOAuthBearerToken(oauthBearerToken OAuthBearerToken) error {
cTokenValue := C.CString(oauthBearerToken.TokenValue)
defer C.free(unsafe.Pointer(cTokenValue))
cPrincipal := C.CString(oauthBearerToken.Principal)
defer C.free(unsafe.Pointer(cPrincipal))
cErrstrSize := C.size_t(512)
cErrstr := (*C.char)(C.malloc(cErrstrSize))
defer C.free(unsafe.Pointer(cErrstr))
cExtensions := make([]*C.char, 2*len(oauthBearerToken.Extensions))
extensionSize := 0
for key, value := range oauthBearerToken.Extensions {
cExtensions[extensionSize] = C.CString(key)
defer C.free(unsafe.Pointer(cExtensions[extensionSize]))
extensionSize++
cExtensions[extensionSize] = C.CString(value)
defer C.free(unsafe.Pointer(cExtensions[extensionSize]))
extensionSize++
}
var cExtensionsToUse **C.char
if extensionSize > 0 {
cExtensionsToUse = (**C.char)(unsafe.Pointer(&cExtensions[0]))
}
cErr := C.rd_kafka_oauthbearer_set_token(h.rk, cTokenValue,
C.int64_t(oauthBearerToken.Expiration.UnixNano()/(1000*1000)), cPrincipal,
cExtensionsToUse, C.size_t(extensionSize), cErrstr, cErrstrSize)
if cErr == C.RD_KAFKA_RESP_ERR_NO_ERROR {
return nil
}
return newErrorFromCString(cErr, cErrstr)
}
// setOauthBearerTokenFailure - see rd_kafka_oauthbearer_set_token_failure()
func (h *handle) setOAuthBearerTokenFailure(errstr string) error {
cerrstr := C.CString(errstr)
defer C.free(unsafe.Pointer(cerrstr))
cErr := C.rd_kafka_oauthbearer_set_token_failure(h.rk, cerrstr)
if cErr == C.RD_KAFKA_RESP_ERR_NO_ERROR {
return nil
}
return newError(cErr)
}
// messageFields controls which fields are made available for producer delivery reports & consumed messages.
// true values indicate that the field should be included
type messageFields struct {
Key bool
Value bool
Headers bool
}
// disableAll disable all fields
func (mf *messageFields) disableAll() {
mf.Key = false
mf.Value = false
mf.Headers = false
}
// newMessageFields returns a new messageFields with all fields enabled
func newMessageFields() *messageFields {
return &messageFields{
Key: true,
Value: true,
Headers: true,
}
}
// newMessageFieldsFrom constructs a new messageFields from the given configuration value
func newMessageFieldsFrom(v ConfigValue) (*messageFields, error) {
msgFields := newMessageFields()
switch v {
case "all":
// nothing to do
case "", "none":
msgFields.disableAll()
default:
msgFields.disableAll()
for _, value := range strings.Split(v.(string), ",") {
switch value {
case "key":
msgFields.Key = true
case "value":
msgFields.Value = true
case "headers":
msgFields.Headers = true
default:
return nil, fmt.Errorf("unknown message field: %s", value)
}
}
}
return msgFields, nil
}

67
vendor/github.com/confluentinc/confluent-kafka-go/kafka/header.go generated vendored

@ -0,0 +1,67 @@
package kafka
/**
* Copyright 2018 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import (
"fmt"
"strconv"
)
/*
#include <string.h>
#include "select_rdkafka.h"
#include "glue_rdkafka.h"
*/
import "C"
// Header represents a single Kafka message header.
//
// Message headers are made up of a list of Header elements, retaining their original insert
// order and allowing for duplicate Keys.
//
// Key is a human readable string identifying the header.
// Value is the key's binary value, Kafka does not put any restrictions on the format of
// of the Value but it should be made relatively compact.
// The value may be a byte array, empty, or nil.
//
// NOTE: Message headers are not available on producer delivery report messages.
type Header struct {
Key string // Header name (utf-8 string)
Value []byte // Header value (nil, empty, or binary)
}
// String returns the Header Key and data in a human representable possibly truncated form
// suitable for displaying to the user.
func (h Header) String() string {
if h.Value == nil {
return fmt.Sprintf("%s=nil", h.Key)
}
valueLen := len(h.Value)
if valueLen == 0 {
return fmt.Sprintf("%s=<empty>", h.Key)
}
truncSize := valueLen
trunc := ""
if valueLen > 50+15 {
truncSize = 50
trunc = fmt.Sprintf("(%d more bytes)", valueLen-truncSize)
}
return fmt.Sprintf("%s=%s%s", h.Key, strconv.Quote(string(h.Value[:truncSize])), trunc)
}

375
vendor/github.com/confluentinc/confluent-kafka-go/kafka/kafka.go generated vendored

@ -0,0 +1,375 @@
/**
* Copyright 2016 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Package kafka provides high-level Apache Kafka producer and consumers
// using bindings on-top of the librdkafka C library.
//
//
// High-level Consumer
//
// * Decide if you want to read messages and events by calling `.Poll()` or
// the deprecated option of using the `.Events()` channel. (If you want to use
// `.Events()` channel then set `"go.events.channel.enable": true`).
//
// * Create a Consumer with `kafka.NewConsumer()` providing at
// least the `bootstrap.servers` and `group.id` configuration properties.
//
// * Call `.Subscribe()` or (`.SubscribeTopics()` to subscribe to multiple topics)
// to join the group with the specified subscription set.
// Subscriptions are atomic, calling `.Subscribe*()` again will leave
// the group and rejoin with the new set of topics.
//
// * Start reading events and messages from either the `.Events` channel
// or by calling `.Poll()`.
//
// * When the group has rebalanced each client member is assigned a
// (sub-)set of topic+partitions.
// By default the consumer will start fetching messages for its assigned
// partitions at this point, but your application may enable rebalance
// events to get an insight into what the assigned partitions where
// as well as set the initial offsets. To do this you need to pass
// `"go.application.rebalance.enable": true` to the `NewConsumer()` call
// mentioned above. You will (eventually) see a `kafka.AssignedPartitions` event
// with the assigned partition set. You can optionally modify the initial
// offsets (they'll default to stored offsets and if there are no previously stored
// offsets it will fall back to `"auto.offset.reset"`
// which defaults to the `latest` message) and then call `.Assign(partitions)`
// to start consuming. If you don't need to modify the initial offsets you will
// not need to call `.Assign()`, the client will do so automatically for you if
// you dont, unless you are using the channel-based consumer in which case
// you MUST call `.Assign()` when receiving the `AssignedPartitions` and
// `RevokedPartitions` events.
//
// * As messages are fetched they will be made available on either the
// `.Events` channel or by calling `.Poll()`, look for event type `*kafka.Message`.
//
// * Handle messages, events and errors to your liking.
//
// * When you are done consuming call `.Close()` to commit final offsets
// and leave the consumer group.
//
//
//
// Producer
//
// * Create a Producer with `kafka.NewProducer()` providing at least
// the `bootstrap.servers` configuration properties.
//
// * Messages may now be produced either by sending a `*kafka.Message`
// on the `.ProduceChannel` or by calling `.Produce()`.
//
// * Producing is an asynchronous operation so the client notifies the application
// of per-message produce success or failure through something called delivery reports.
// Delivery reports are by default emitted on the `.Events()` channel as `*kafka.Message`
// and you should check `msg.TopicPartition.Error` for `nil` to find out if the message
// was succesfully delivered or not.
// It is also possible to direct delivery reports to alternate channels
// by providing a non-nil `chan Event` channel to `.Produce()`.
// If no delivery reports are wanted they can be completely disabled by
// setting configuration property `"go.delivery.reports": false`.
//
// * When you are done producing messages you will need to make sure all messages
// are indeed delivered to the broker (or failed), remember that this is
// an asynchronous client so some of your messages may be lingering in internal
// channels or tranmission queues.
// To do this you can either keep track of the messages you've produced
// and wait for their corresponding delivery reports, or call the convenience
// function `.Flush()` that will block until all message deliveries are done
// or the provided timeout elapses.
//
// * Finally call `.Close()` to decommission the producer.
//
//
// Transactional producer API
//
// The transactional producer operates on top of the idempotent producer,
// and provides full exactly-once semantics (EOS) for Apache Kafka when used
// with the transaction aware consumer (`isolation.level=read_committed`).
//
// A producer instance is configured for transactions by setting the
// `transactional.id` to an identifier unique for the application. This
// id will be used to fence stale transactions from previous instances of
// the application, typically following an outage or crash.
//
// After creating the transactional producer instance using `NewProducer()`
// the transactional state must be initialized by calling
// `InitTransactions()`. This is a blocking call that will
// acquire a runtime producer id from the transaction coordinator broker
// as well as abort any stale transactions and fence any still running producer
// instances with the same `transactional.id`.
//
// Once transactions are initialized the application may begin a new
// transaction by calling `BeginTransaction()`.
// A producer instance may only have one single on-going transaction.
//
// Any messages produced after the transaction has been started will
// belong to the ongoing transaction and will be committed or aborted
// atomically.
// It is not permitted to produce messages outside a transaction
// boundary, e.g., before `BeginTransaction()` or after `CommitTransaction()`,
// `AbortTransaction()` or if the current transaction has failed.
//
// If consumed messages are used as input to the transaction, the consumer
// instance must be configured with `enable.auto.commit` set to `false`.
// To commit the consumed offsets along with the transaction pass the
// list of consumed partitions and the last offset processed + 1 to
// `SendOffsetsToTransaction()` prior to committing the transaction.
// This allows an aborted transaction to be restarted using the previously
// committed offsets.
//
// To commit the produced messages, and any consumed offsets, to the
// current transaction, call `CommitTransaction()`.
// This call will block until the transaction has been fully committed or
// failed (typically due to fencing by a newer producer instance).
//
// Alternatively, if processing fails, or an abortable transaction error is
// raised, the transaction needs to be aborted by calling
// `AbortTransaction()` which marks any produced messages and
// offset commits as aborted.
//
// After the current transaction has been committed or aborted a new
// transaction may be started by calling `BeginTransaction()` again.
//
// Retriable errors:
// Some error cases allow the attempted operation to be retried, this is
// indicated by the error object having the retriable flag set which can
// be detected by calling `err.(kafka.Error).IsRetriable()`.
// When this flag is set the application may retry the operation immediately
// or preferably after a shorter grace period (to avoid busy-looping).
// Retriable errors include timeouts, broker transport failures, etc.
//
// Abortable errors:
// An ongoing transaction may fail permanently due to various errors,
// such as transaction coordinator becoming unavailable, write failures to the
// Apache Kafka log, under-replicated partitions, etc.
// At this point the producer application must abort the current transaction
// using `AbortTransaction()` and optionally start a new transaction
// by calling `BeginTransaction()`.
// Whether an error is abortable or not is detected by calling
// `err.(kafka.Error).TxnRequiresAbort()` on the returned error object.
//
// Fatal errors:
// While the underlying idempotent producer will typically only raise
// fatal errors for unrecoverable cluster errors where the idempotency
// guarantees can't be maintained, most of these are treated as abortable by
// the transactional producer since transactions may be aborted and retried
// in their entirety;
// The transactional producer on the other hand introduces a set of additional
// fatal errors which the application needs to handle by shutting down the
// producer and terminate. There is no way for a producer instance to recover
// from fatal errors.
// Whether an error is fatal or not is detected by calling
// `err.(kafka.Error).IsFatal()` on the returned error object or by checking
// the global `GetFatalError()`.
//
// Handling of other errors:
// For errors that have neither retriable, abortable or the fatal flag set
// it is not always obvious how to handle them. While some of these errors
// may be indicative of bugs in the application code, such as when
// an invalid parameter is passed to a method, other errors might originate
// from the broker and be passed thru as-is to the application.
// The general recommendation is to treat these errors, that have
// neither the retriable or abortable flags set, as fatal.
//
// Error handling example:
// retry:
//
// err := producer.CommitTransaction(...)
// if err == nil {
// return nil
// } else if err.(kafka.Error).TxnRequiresAbort() {
// do_abort_transaction_and_reset_inputs()
// } else if err.(kafka.Error).IsRetriable() {
// goto retry
// } else { // treat all other errors as fatal errors
// panic(err)
// }
//
//
// Events
//
// Apart from emitting messages and delivery reports the client also communicates
// with the application through a number of different event types.
// An application may choose to handle or ignore these events.
//
// Consumer events
//
// * `*kafka.Message` - a fetched message.
//
// * `AssignedPartitions` - The assigned partition set for this client following a rebalance.
// Requires `go.application.rebalance.enable`
//
// * `RevokedPartitions` - The counter part to `AssignedPartitions` following a rebalance.
// `AssignedPartitions` and `RevokedPartitions` are symmetrical.
// Requires `go.application.rebalance.enable`
//
// * `PartitionEOF` - Consumer has reached the end of a partition.
// NOTE: The consumer will keep trying to fetch new messages for the partition.
//
// * `OffsetsCommitted` - Offset commit results (when `enable.auto.commit` is enabled).
//
//
// Producer events
//
// * `*kafka.Message` - delivery report for produced message.
// Check `.TopicPartition.Error` for delivery result.
//
//
// Generic events for both Consumer and Producer
//
// * `KafkaError` - client (error codes are prefixed with _) or broker error.
// These errors are normally just informational since the
// client will try its best to automatically recover (eventually).
//
// * `OAuthBearerTokenRefresh` - retrieval of a new SASL/OAUTHBEARER token is required.
// This event only occurs with sasl.mechanism=OAUTHBEARER.
// Be sure to invoke SetOAuthBearerToken() on the Producer/Consumer/AdminClient
// instance when a successful token retrieval is completed, otherwise be sure to
// invoke SetOAuthBearerTokenFailure() to indicate that retrieval failed (or
// if setting the token failed, which could happen if an extension doesn't meet
// the required regular expression); invoking SetOAuthBearerTokenFailure() will
// schedule a new event for 10 seconds later so another retrieval can be attempted.
//
//
// Hint: If your application registers a signal notification
// (signal.Notify) makes sure the signals channel is buffered to avoid
// possible complications with blocking Poll() calls.
//
// Note: The Confluent Kafka Go client is safe for concurrent use.
package kafka
import (
"fmt"
// Make sure librdkafka_vendor/ sub-directory is included in vendor pulls.
_ "github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor"
"unsafe"
)
/*
#include <stdlib.h>
#include <string.h>
#include "select_rdkafka.h"
static rd_kafka_topic_partition_t *_c_rdkafka_topic_partition_list_entry(rd_kafka_topic_partition_list_t *rktparlist, int idx) {
return idx < rktparlist->cnt ? &rktparlist->elems[idx] : NULL;
}
*/
import "C"
// PartitionAny represents any partition (for partitioning),
// or unspecified value (for all other cases)
const PartitionAny = int32(C.RD_KAFKA_PARTITION_UA)
// TopicPartition is a generic placeholder for a Topic+Partition and optionally Offset.
type TopicPartition struct {
Topic *string
Partition int32
Offset Offset
Metadata *string
Error error
}
func (p TopicPartition) String() string {
topic := "<null>"
if p.Topic != nil {
topic = *p.Topic
}
if p.Error != nil {
return fmt.Sprintf("%s[%d]@%s(%s)",
topic, p.Partition, p.Offset, p.Error)
}
return fmt.Sprintf("%s[%d]@%s",
topic, p.Partition, p.Offset)
}
// TopicPartitions is a slice of TopicPartitions that also implements
// the sort interface
type TopicPartitions []TopicPartition
func (tps TopicPartitions) Len() int {
return len(tps)
}
func (tps TopicPartitions) Less(i, j int) bool {
if *tps[i].Topic < *tps[j].Topic {
return true
} else if *tps[i].Topic > *tps[j].Topic {
return false
}
return tps[i].Partition < tps[j].Partition
}
func (tps TopicPartitions) Swap(i, j int) {
tps[i], tps[j] = tps[j], tps[i]
}
// new_cparts_from_TopicPartitions creates a new C rd_kafka_topic_partition_list_t
// from a TopicPartition array.
func newCPartsFromTopicPartitions(partitions []TopicPartition) (cparts *C.rd_kafka_topic_partition_list_t) {
cparts = C.rd_kafka_topic_partition_list_new(C.int(len(partitions)))
for _, part := range partitions {
ctopic := C.CString(*part.Topic)
defer C.free(unsafe.Pointer(ctopic))
rktpar := C.rd_kafka_topic_partition_list_add(cparts, ctopic, C.int32_t(part.Partition))
rktpar.offset = C.int64_t(part.Offset)
if part.Metadata != nil {
cmetadata := C.CString(*part.Metadata)
rktpar.metadata = unsafe.Pointer(cmetadata)
rktpar.metadata_size = C.size_t(len(*part.Metadata))
}
}
return cparts
}
func setupTopicPartitionFromCrktpar(partition *TopicPartition, crktpar *C.rd_kafka_topic_partition_t) {
topic := C.GoString(crktpar.topic)
partition.Topic = &topic
partition.Partition = int32(crktpar.partition)
partition.Offset = Offset(crktpar.offset)
if crktpar.metadata_size > 0 {
size := C.int(crktpar.metadata_size)
cstr := (*C.char)(unsafe.Pointer(crktpar.metadata))
metadata := C.GoStringN(cstr, size)
partition.Metadata = &metadata
}
if crktpar.err != C.RD_KAFKA_RESP_ERR_NO_ERROR {
partition.Error = newError(crktpar.err)
}
}
func newTopicPartitionsFromCparts(cparts *C.rd_kafka_topic_partition_list_t) (partitions []TopicPartition) {
partcnt := int(cparts.cnt)
partitions = make([]TopicPartition, partcnt)
for i := 0; i < partcnt; i++ {
crktpar := C._c_rdkafka_topic_partition_list_entry(cparts, C.int(i))
setupTopicPartitionFromCrktpar(&partitions[i], crktpar)
}
return partitions
}
// LibraryVersion returns the underlying librdkafka library version as a
// (version_int, version_str) tuple.
func LibraryVersion() (int, string) {
ver := (int)(C.rd_kafka_version())
verstr := C.GoString(C.rd_kafka_version_str())
return ver, verstr
}

3
vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/.gitignore generated vendored

@ -0,0 +1,3 @@
*.tar.gz
*.tgz
tmp*

366
vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/LICENSES.txt generated vendored

@ -0,0 +1,366 @@
LICENSE
--------------------------------------------------------------
librdkafka - Apache Kafka C driver library
Copyright (c) 2012-2020, Magnus Edenhill
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
LICENSE.crc32c
--------------------------------------------------------------
# For src/crc32c.c copied (with modifications) from
# http://stackoverflow.com/a/17646775/1821055
/* crc32c.c -- compute CRC-32C using the Intel crc32 instruction
* Copyright (C) 2013 Mark Adler
* Version 1.1 1 Aug 2013 Mark Adler
*/
/*
This software is provided 'as-is', without any express or implied
warranty. In no event will the author be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must not
claim that you wrote the original software. If you use this software
in a product, an acknowledgment in the product documentation would be
appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
misrepresented as being the original software.
3. This notice may not be removed or altered from any source distribution.
Mark Adler
madler@alumni.caltech.edu
*/
LICENSE.fnv1a
--------------------------------------------------------------
parts of src/rdfnv1a.c: http://www.isthe.com/chongo/src/fnv/hash_32a.c
Please do not copyright this code. This code is in the public domain.
LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO
EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR
CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
By:
chongo <Landon Curt Noll> /\oo/\
http://www.isthe.com/chongo/
Share and Enjoy! :-)
LICENSE.hdrhistogram
--------------------------------------------------------------
This license covers src/rdhdrhistogram.c which is a C port of
Coda Hale's Golang HdrHistogram https://github.com/codahale/hdrhistogram
at revision 3a0bb77429bd3a61596f5e8a3172445844342120
-----------------------------------------------------------------------------
The MIT License (MIT)
Copyright (c) 2014 Coda Hale
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE
LICENSE.lz4
--------------------------------------------------------------
src/rdxxhash.[ch] src/lz4*.[ch]: git@github.com:lz4/lz4.git e2827775ee80d2ef985858727575df31fc60f1f3
LZ4 Library
Copyright (c) 2011-2016, Yann Collet
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
LICENSE.murmur2
--------------------------------------------------------------
parts of src/rdmurmur2.c: git@github.com:abrandoned/murmur2.git
MurMurHash2 Library
//-----------------------------------------------------------------------------
// MurmurHash2 was written by Austin Appleby, and is placed in the public
// domain. The author hereby disclaims copyright to this source code.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
LICENSE.pycrc
--------------------------------------------------------------
The following license applies to the files rdcrc32.c and rdcrc32.h which
have been generated by the pycrc tool.
============================================================================
Copyright (c) 2006-2012, Thomas Pircher <tehpeh@gmx.net>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
LICENSE.queue
--------------------------------------------------------------
For sys/queue.h:
* Copyright (c) 1991, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)queue.h 8.5 (Berkeley) 8/20/94
* $FreeBSD$
LICENSE.regexp
--------------------------------------------------------------
regexp.c and regexp.h from https://github.com/ccxvii/minilibs sha 875c33568b5a4aa4fb3dd0c52ea98f7f0e5ca684
"
These libraries are in the public domain (or the equivalent where that is not possible). You can do anything you want with them. You have no legal obligation to do anything else, although I appreciate attribution.
"
LICENSE.snappy
--------------------------------------------------------------
######################################################################
# LICENSE.snappy covers files: snappy.c, snappy.h, snappy_compat.h #
# originally retrieved from http://github.com/andikleen/snappy-c #
# git revision 8015f2d28739b9a6076ebaa6c53fe27bc238d219 #
######################################################################
The snappy-c code is under the same license as the original snappy source
Copyright 2011 Intel Corporation All Rights Reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Intel Corporation nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
LICENSE.tinycthread
--------------------------------------------------------------
From https://github.com/tinycthread/tinycthread/README.txt c57166cd510ffb5022dd5f127489b131b61441b9
License
-------
Copyright (c) 2012 Marcus Geelnard
2013-2014 Evan Nemerson
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must not
claim that you wrote the original software. If you use this software
in a product, an acknowledgment in the product documentation would be
appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
misrepresented as being the original software.
3. This notice may not be removed or altered from any source
distribution.
LICENSE.wingetopt
--------------------------------------------------------------
For the files wingetopt.c wingetopt.h downloaded from https://github.com/alex85k/wingetopt
/*
* Copyright (c) 2002 Todd C. Miller <Todd.Miller@courtesan.com>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*
* Sponsored in part by the Defense Advanced Research Projects
* Agency (DARPA) and Air Force Research Laboratory, Air Force
* Materiel Command, USAF, under agreement number F39502-99-1-0512.
*/
/*-
* Copyright (c) 2000 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Dieter Baron and Thomas Klausner.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

24
vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/README.md generated vendored

@ -0,0 +1,24 @@
# Bundling prebuilt librdkafka
confluent-kafka-go bundles prebuilt statically linked
versions of librdkafka for the following platforms:
* MacOSX x64 (aka Darwin)
* Linux glibc x64 (Ubuntu, CentOS, etc)
* Linux musl x64 (Alpine)
## Import static librdkafka bundle
First create the static librdkafka bundle following the instructions in
librdkafka's packaging/nuget/README.md.
Then import the new version by using the import.sh script here, this script
will create a branch, import the bundle, create a commit and push the
branch to Github for PR review. This PR must be manually opened, reviewed
and then finally merged (make sure to merge it, DO NOT squash or rebase).
$ ./import.sh ~/path/to/librdkafka-static-bundle-v1.4.0.tgz
This will copy the static library and the rdkafka.h header file
to this directory, as well as generate a new ../build_..go file
for this platform + variant.

113
vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/bundle-import.sh generated vendored

@ -0,0 +1,113 @@
#!/bin/bash
#
# Updates the bundled prebuilt librdkafka libraries to specified version.
#
set -e
usage() {
echo "Usage: $0 librdkafka-static-bundle-<VERSION>.tgz"
echo ""
echo "This tool must be run from the TOPDIR/kafka/librdkafka_vendor directory"
exit 1
}
parse_dynlibs() {
# Parse dynamic libraries from pkg-config file,
# both the ones specified with Libs: but also through Requires:
local pc=$1
local libs=
local req=
local n=
for req in $(grep ^Requires: $pc | sed -e 's/^Requires://'); do
n=$(pkg-config --libs $req)
if [[ $n == -l* ]]; then
libs="${libs} $n"
fi
done
for n in $(grep ^Libs: $pc); do
if [[ $n == -l* ]]; then
libs="${libs} $n"
fi
done
echo "$libs"
}
setup_build() {
# Copies static library from the temp directory into final location,
# extracts dynamic lib list from the pkg-config file,
# and generates the build_..go file
local btype=$1
local apath=$2
local pc=$3
local srcinfo=$4
local build_tag=
local gpath="../build_${btype}.go"
local dpath="librdkafka_${btype}.a"
if [[ $btype == glibc_linux ]]; then
build_tag="// +build !musl"
elif [[ $btype == musl_linux ]]; then
build_tag="// +build musl"
fi
local dynlibs=$(parse_dynlibs $pc)
echo "Copying $apath to $dpath"
cp "$apath" "$dpath"
echo "Generating $gpath (extra build tag: $build_tag)"
cat >$gpath <<EOF
// +build !dynamic
$build_tag
// This file was auto-generated by librdkafka_vendor/bundle-import.sh, DO NOT EDIT.
package kafka
// #cgo CFLAGS: -DUSE_VENDORED_LIBRDKAFKA -DLIBRDKAFKA_STATICLIB
// #cgo LDFLAGS: \${SRCDIR}/librdkafka_vendor/${dpath} $dynlibs
import "C"
// LibrdkafkaLinkInfo explains how librdkafka was linked to the Go client
const LibrdkafkaLinkInfo = "static ${btype} from ${srcinfo}"
EOF
git add "$dpath" "$gpath"
}
bundle="$1"
[[ -f $bundle ]] || usage
bundlename=$(basename "$bundle")
bdir=$(mktemp -d tmpXXXXXX)
echo "Extracting bundle $bundle:"
tar -xzvf "$bundle" -C "$bdir/"
echo "Copying librdkafka files"
for f in rdkafka.h LICENSES.txt ; do
cp $bdir/$f . || true
git add "$f"
done
for btype in glibc_linux musl_linux darwin windows ; do
lib=$bdir/librdkafka_${btype}.a
pc=${lib/%.a/.pc}
[[ -f $lib ]] || (echo "Expected file $lib missing" ; exit 1)
[[ -f $pc ]] || (echo "Expected file $pc missing" ; exit 1)
setup_build $btype $lib $pc $bundlename
done
rm -rf "$bdir"
echo "All done"

110
vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/import.sh generated vendored

@ -0,0 +1,110 @@
#!/bin/bash
#
#
# Import a new version of librdkafka based on a librdkafka static bundle.
# This will create a separate branch, import librdkafka, make a commit,
# and then ask you to push the branch to github, have it reviewed,
# and then later merged (NOT squashed or rebased).
# Having a merge per import allows future shallow clones to skip and ignore
# older imports, hopefully reducing the amount of git history data 'go get'
# needs to download.
set -e
usage() {
echo "Usage: $0 [--devel] path/to/librdkafka-static-bundle-<VERSION>.tgz"
echo ""
echo "This tool must be run from the TOPDIR/kafka/librdkafka directory"
echo ""
echo "Options:"
echo " --devel - Development use: No branch checks and does not push to github"
exit 1
}
error_cleanup() {
echo "Error occurred, cleaning up"
git checkout $currbranch
git branch -D $import_branch
exit 1
}
devel=0
if [[ $1 == --devel ]]; then
devel=1
shift
fi
bundle="$1"
[[ -f $bundle ]] || usage
# Parse the librdkafka version from the bundle
bundlename=$(basename $bundle)
version=${bundlename#librdkafka-static-bundle-}
version=${version%.tgz}
if [[ -z $version ]]; then
echo "Error: Could not parse version from bundle $bundle"
exit 1
fi
# Verify branch state
curr_branch=$(git symbolic-ref HEAD 2>/dev/null | cut -d"/" -f 3-)
uncommitted=$(git status --untracked-files=no --porcelain)
if [[ $devel != 1 ]] && ( [[ $curr_branch != master ]] || [[ ! -z $uncommitted ]] ); then
echo "Error: This script must be run on an up-to-date, clean, master branch"
if [[ ! -z $uncommitted ]]; then
echo "Uncommitted files:"
echo "$uncommitted"
fi
exit 1
fi
# Create import branch, import bundle, commit.
import_branch="import_$version"
exists=$(git branch -rlq | grep "/$import_branch\$" || true)
if [[ ! -z $exists ]]; then
echo "Error: This version branch already seems to exist: $exists: already imorted?"
[[ $devel != 1 ]] && exit 1
fi
echo "Checking for existing commits that match this version (should be none)"
git log --oneline | grep "^librdkafka static bundle $version\$" && exit 1
echo "Creating import branch $import_branch"
git checkout -b $import_branch
echo "Importing bundle $bundle"
./bundle-import.sh "$bundle" || error_cleanup
echo "Committing $version"
git commit -a -m "librdkafka static bundle $version" || error_cleanup
echo "Updating error codes and docs"
pushd ../../
make -f mk/Makefile docs || error_cleanup
git commit -a -m "Documentation and error code update for librdkafka $version" \
|| error_cleanup
popd
if [[ $devel != 1 ]]; then
echo "Pushing branch"
git push origin $import_branch || error_cleanup
fi
git checkout $curr_branch
if [[ $devel != 1 ]]; then
git branch -D $import_branch
fi
echo ""
echo "############## IMPORT OF $version COMPLETE ##############"
if [[ $devel != 1 ]]; then
echo "Branch $import_branch has been pushed."
echo "Create a PR, have it reviewed and then merge it (do NOT squash or rebase)."
fi

21
vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/librdkafka.go generated vendored

@ -0,0 +1,21 @@
/**
* Copyright 2020 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package librdkafka
// LibrdkafkaGoSubdir is a dummy variable needed to export something so the
// file is not empty.
var LibrdkafkaGoSubdir = true

BIN
vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/librdkafka_darwin.a generated vendored

Binary file not shown.

BIN
vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/librdkafka_glibc_linux.a generated vendored

Binary file not shown.

BIN
vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/librdkafka_musl_linux.a generated vendored

Binary file not shown.

BIN
vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/librdkafka_windows.a generated vendored

Binary file not shown.

7781
vendor/github.com/confluentinc/confluent-kafka-go/kafka/librdkafka_vendor/rdkafka.h generated vendored

File diff suppressed because it is too large Load Diff

89
vendor/github.com/confluentinc/confluent-kafka-go/kafka/log.go generated vendored

@ -0,0 +1,89 @@
package kafka
import (
"fmt"
"time"
)
/*
#include "select_rdkafka.h"
*/
import "C"
// LogEvent represent the log from librdkafka internal log queue
type LogEvent struct {
Name string // Name of client instance
Tag string // Log tag that provides context to the log Message (e.g., "METADATA" or "GRPCOORD")
Message string // Log message
Level int // Log syslog level, lower is more critical.
Timestamp time.Time // Log timestamp
}
// newLogEvent creates a new LogEvent from the given rd_kafka_event_t.
//
// This function does not take ownership of the cEvent pointer. You need to
// free its resources using C.rd_kafka_event_destroy afterwards.
//
// The cEvent object needs to be of type C.RD_KAFKA_EVENT_LOG. Calling this
// function with an object of another type has undefined behaviour.
func (h *handle) newLogEvent(cEvent *C.rd_kafka_event_t) LogEvent {
var tag, message *C.char
var level C.int
C.rd_kafka_event_log(cEvent, &(tag), &(message), &(level))
return LogEvent{
Name: h.name,
Tag: C.GoString(tag),
Message: C.GoString(message),
Level: int(level),
Timestamp: time.Now(),
}
}
// pollLogEvents polls log events from librdkafka and pushes them to toChannel,
// until doneChan is closed.
//
// Each call to librdkafka times out after timeoutMs. If a call to librdkafka
// is ongoing when doneChan is closed, the function will wait until the call
// returns or times out, whatever happens first.
func (h *handle) pollLogEvents(toChannel chan LogEvent, timeoutMs int, doneChan chan bool) {
for {
select {
case <-doneChan:
return
default:
cEvent := C.rd_kafka_queue_poll(h.logq, C.int(timeoutMs))
if cEvent == nil {
continue
}
if C.rd_kafka_event_type(cEvent) != C.RD_KAFKA_EVENT_LOG {
C.rd_kafka_event_destroy(cEvent)
continue
}
logEvent := h.newLogEvent(cEvent)
C.rd_kafka_event_destroy(cEvent)
select {
case <-doneChan:
return
case toChannel <- logEvent:
continue
}
}
}
}
func (logEvent LogEvent) String() string {
return fmt.Sprintf(
"[%v][%s][%s][%d]%s",
logEvent.Timestamp.Format(time.RFC3339),
logEvent.Name,
logEvent.Tag,
logEvent.Level,
logEvent.Message)
}

223
vendor/github.com/confluentinc/confluent-kafka-go/kafka/message.go generated vendored

@ -0,0 +1,223 @@
package kafka
/**
* Copyright 2016 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import (
"fmt"
"time"
"unsafe"
)
/*
#include <string.h>
#include <stdlib.h>
#include "select_rdkafka.h"
#include "glue_rdkafka.h"
void setup_rkmessage (rd_kafka_message_t *rkmessage,
rd_kafka_topic_t *rkt, int32_t partition,
const void *payload, size_t len,
void *key, size_t keyLen, void *opaque) {
rkmessage->rkt = rkt;
rkmessage->partition = partition;
rkmessage->payload = (void *)payload;
rkmessage->len = len;
rkmessage->key = (void *)key;
rkmessage->key_len = keyLen;
rkmessage->_private = opaque;
}
*/
import "C"
// TimestampType is a the Message timestamp type or source
//
type TimestampType int
const (
// TimestampNotAvailable indicates no timestamp was set, or not available due to lacking broker support
TimestampNotAvailable = TimestampType(C.RD_KAFKA_TIMESTAMP_NOT_AVAILABLE)
// TimestampCreateTime indicates timestamp set by producer (source time)
TimestampCreateTime = TimestampType(C.RD_KAFKA_TIMESTAMP_CREATE_TIME)
// TimestampLogAppendTime indicates timestamp set set by broker (store time)
TimestampLogAppendTime = TimestampType(C.RD_KAFKA_TIMESTAMP_LOG_APPEND_TIME)
)
func (t TimestampType) String() string {
switch t {
case TimestampCreateTime:
return "CreateTime"
case TimestampLogAppendTime:
return "LogAppendTime"
case TimestampNotAvailable:
fallthrough
default:
return "NotAvailable"
}
}
// Message represents a Kafka message
type Message struct {
TopicPartition TopicPartition
Value []byte
Key []byte
Timestamp time.Time
TimestampType TimestampType
Opaque interface{}
Headers []Header
}
// String returns a human readable representation of a Message.
// Key and payload are not represented.
func (m *Message) String() string {
var topic string
if m.TopicPartition.Topic != nil {
topic = *m.TopicPartition.Topic
} else {
topic = ""
}
return fmt.Sprintf("%s[%d]@%s", topic, m.TopicPartition.Partition, m.TopicPartition.Offset)
}
func (h *handle) getRktFromMessage(msg *Message) (crkt *C.rd_kafka_topic_t) {
if msg.TopicPartition.Topic == nil {
return nil
}
return h.getRkt(*msg.TopicPartition.Topic)
}
// setupHeadersFromGlueMsg converts the C tmp headers in gMsg to
// Go Headers in msg.
// gMsg.tmphdrs will be freed.
func setupHeadersFromGlueMsg(msg *Message, gMsg *C.glue_msg_t) {
msg.Headers = make([]Header, gMsg.tmphdrsCnt)
for n := range msg.Headers {
tmphdr := (*[1 << 30]C.tmphdr_t)(unsafe.Pointer(gMsg.tmphdrs))[n]
msg.Headers[n].Key = C.GoString(tmphdr.key)
if tmphdr.val != nil {
msg.Headers[n].Value = C.GoBytes(unsafe.Pointer(tmphdr.val), C.int(tmphdr.size))
} else {
msg.Headers[n].Value = nil
}
}
C.free(unsafe.Pointer(gMsg.tmphdrs))
}
func (h *handle) newMessageFromGlueMsg(gMsg *C.glue_msg_t) (msg *Message) {
msg = &Message{}
if gMsg.ts != -1 {
ts := int64(gMsg.ts)
msg.TimestampType = TimestampType(gMsg.tstype)
msg.Timestamp = time.Unix(ts/1000, (ts%1000)*1000000)
}
if gMsg.tmphdrsCnt > 0 {
setupHeadersFromGlueMsg(msg, gMsg)
}
h.setupMessageFromC(msg, gMsg.msg)
return msg
}
// setupMessageFromC sets up a message object from a C rd_kafka_message_t
func (h *handle) setupMessageFromC(msg *Message, cmsg *C.rd_kafka_message_t) {
if cmsg.rkt != nil {
topic := h.getTopicNameFromRkt(cmsg.rkt)
msg.TopicPartition.Topic = &topic
}
msg.TopicPartition.Partition = int32(cmsg.partition)
if cmsg.payload != nil && h.msgFields.Value {
msg.Value = C.GoBytes(unsafe.Pointer(cmsg.payload), C.int(cmsg.len))
}
if cmsg.key != nil && h.msgFields.Key {
msg.Key = C.GoBytes(unsafe.Pointer(cmsg.key), C.int(cmsg.key_len))
}
if h.msgFields.Headers {
var gMsg C.glue_msg_t
gMsg.msg = cmsg
gMsg.want_hdrs = C.int8_t(1)
chdrsToTmphdrs(&gMsg)
if gMsg.tmphdrsCnt > 0 {
setupHeadersFromGlueMsg(msg, &gMsg)
}
}
msg.TopicPartition.Offset = Offset(cmsg.offset)
if cmsg.err != 0 {
msg.TopicPartition.Error = newError(cmsg.err)
}
}
// newMessageFromC creates a new message object from a C rd_kafka_message_t
// NOTE: For use with Producer: does not set message timestamp fields.
func (h *handle) newMessageFromC(cmsg *C.rd_kafka_message_t) (msg *Message) {
msg = &Message{}
h.setupMessageFromC(msg, cmsg)
return msg
}
// messageToC sets up cmsg as a clone of msg
func (h *handle) messageToC(msg *Message, cmsg *C.rd_kafka_message_t) {
var valp unsafe.Pointer
var keyp unsafe.Pointer
// to circumvent Cgo constraints we need to allocate C heap memory
// for both Value and Key (one allocation back to back)
// and copy the bytes from Value and Key to the C memory.
// We later tell librdkafka (in produce()) to free the
// C memory pointer when it is done.
var payload unsafe.Pointer
valueLen := 0
keyLen := 0
if msg.Value != nil {
valueLen = len(msg.Value)
}
if msg.Key != nil {
keyLen = len(msg.Key)
}
allocLen := valueLen + keyLen
if allocLen > 0 {
payload = C.malloc(C.size_t(allocLen))
if valueLen > 0 {
copy((*[1 << 30]byte)(payload)[0:valueLen], msg.Value)
valp = payload
}
if keyLen > 0 {
copy((*[1 << 30]byte)(payload)[valueLen:allocLen], msg.Key)
keyp = unsafe.Pointer(&((*[1 << 31]byte)(payload)[valueLen]))
}
}
cmsg.rkt = h.getRktFromMessage(msg)
cmsg.partition = C.int32_t(msg.TopicPartition.Partition)
cmsg.payload = valp
cmsg.len = C.size_t(valueLen)
cmsg.key = keyp
cmsg.key_len = C.size_t(keyLen)
cmsg._private = nil
}
// used for testing messageToC performance
func (h *handle) messageToCDummy(msg *Message) {
var cmsg C.rd_kafka_message_t
h.messageToC(msg, &cmsg)
}

180
vendor/github.com/confluentinc/confluent-kafka-go/kafka/metadata.go generated vendored

@ -0,0 +1,180 @@
/**
* Copyright 2016 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package kafka
import (
"unsafe"
)
/*
#include <stdlib.h>
#include "select_rdkafka.h"
struct rd_kafka_metadata_broker *_getMetadata_broker_element(struct rd_kafka_metadata *m, int i) {
return &m->brokers[i];
}
struct rd_kafka_metadata_topic *_getMetadata_topic_element(struct rd_kafka_metadata *m, int i) {
return &m->topics[i];
}
struct rd_kafka_metadata_partition *_getMetadata_partition_element(struct rd_kafka_metadata *m, int topic_idx, int partition_idx) {
return &m->topics[topic_idx].partitions[partition_idx];
}
int32_t _get_int32_element (int32_t *arr, int i) {
return arr[i];
}
*/
import "C"
// BrokerMetadata contains per-broker metadata
type BrokerMetadata struct {
ID int32
Host string
Port int
}
// PartitionMetadata contains per-partition metadata
type PartitionMetadata struct {
ID int32
Error Error
Leader int32
Replicas []int32
Isrs []int32
}
// TopicMetadata contains per-topic metadata
type TopicMetadata struct {
Topic string
Partitions []PartitionMetadata
Error Error
}
// Metadata contains broker and topic metadata for all (matching) topics
type Metadata struct {
Brokers []BrokerMetadata
Topics map[string]TopicMetadata
OriginatingBroker BrokerMetadata
}
// getMetadata queries broker for cluster and topic metadata.
// If topic is non-nil only information about that topic is returned, else if
// allTopics is false only information about locally used topics is returned,
// else information about all topics is returned.
func getMetadata(H Handle, topic *string, allTopics bool, timeoutMs int) (*Metadata, error) {
h := H.gethandle()
var rkt *C.rd_kafka_topic_t
if topic != nil {
rkt = h.getRkt(*topic)
}
var cMd *C.struct_rd_kafka_metadata
cErr := C.rd_kafka_metadata(h.rk, bool2cint(allTopics),
rkt, &cMd, C.int(timeoutMs))
if cErr != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return nil, newError(cErr)
}
m := Metadata{}
defer C.rd_kafka_metadata_destroy(cMd)
m.Brokers = make([]BrokerMetadata, cMd.broker_cnt)
for i := 0; i < int(cMd.broker_cnt); i++ {
b := C._getMetadata_broker_element(cMd, C.int(i))
m.Brokers[i] = BrokerMetadata{int32(b.id), C.GoString(b.host),
int(b.port)}
}
m.Topics = make(map[string]TopicMetadata, int(cMd.topic_cnt))
for i := 0; i < int(cMd.topic_cnt); i++ {
t := C._getMetadata_topic_element(cMd, C.int(i))
thisTopic := C.GoString(t.topic)
m.Topics[thisTopic] = TopicMetadata{Topic: thisTopic,
Error: newError(t.err),
Partitions: make([]PartitionMetadata, int(t.partition_cnt))}
for j := 0; j < int(t.partition_cnt); j++ {
p := C._getMetadata_partition_element(cMd, C.int(i), C.int(j))
m.Topics[thisTopic].Partitions[j] = PartitionMetadata{
ID: int32(p.id),
Error: newError(p.err),
Leader: int32(p.leader)}
m.Topics[thisTopic].Partitions[j].Replicas = make([]int32, int(p.replica_cnt))
for ir := 0; ir < int(p.replica_cnt); ir++ {
m.Topics[thisTopic].Partitions[j].Replicas[ir] = int32(C._get_int32_element(p.replicas, C.int(ir)))
}
m.Topics[thisTopic].Partitions[j].Isrs = make([]int32, int(p.isr_cnt))
for ii := 0; ii < int(p.isr_cnt); ii++ {
m.Topics[thisTopic].Partitions[j].Isrs[ii] = int32(C._get_int32_element(p.isrs, C.int(ii)))
}
}
}
m.OriginatingBroker = BrokerMetadata{int32(cMd.orig_broker_id),
C.GoString(cMd.orig_broker_name), 0}
return &m, nil
}
// queryWatermarkOffsets returns the broker's low and high offsets for the given topic
// and partition.
func queryWatermarkOffsets(H Handle, topic string, partition int32, timeoutMs int) (low, high int64, err error) {
h := H.gethandle()
ctopic := C.CString(topic)
defer C.free(unsafe.Pointer(ctopic))
var cLow, cHigh C.int64_t
e := C.rd_kafka_query_watermark_offsets(h.rk, ctopic, C.int32_t(partition),
&cLow, &cHigh, C.int(timeoutMs))
if e != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return 0, 0, newError(e)
}
low = int64(cLow)
high = int64(cHigh)
return low, high, nil
}
// getWatermarkOffsets returns the clients cached low and high offsets for the given topic
// and partition.
func getWatermarkOffsets(H Handle, topic string, partition int32) (low, high int64, err error) {
h := H.gethandle()
ctopic := C.CString(topic)
defer C.free(unsafe.Pointer(ctopic))
var cLow, cHigh C.int64_t
e := C.rd_kafka_get_watermark_offsets(h.rk, ctopic, C.int32_t(partition),
&cLow, &cHigh)
if e != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return 0, 0, newError(e)
}
low = int64(cLow)
high = int64(cHigh)
return low, high, nil
}

35
vendor/github.com/confluentinc/confluent-kafka-go/kafka/misc.go generated vendored

@ -0,0 +1,35 @@
/**
* Copyright 2016 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package kafka
import "C"
// bool2int converts a bool to a C.int (1 or 0)
func bool2cint(b bool) C.int {
if b {
return 1
}
return 0
}
// cint2bool converts a C.int to a bool
func cint2bool(v C.int) bool {
if v == 0 {
return false
}
return true
}

145
vendor/github.com/confluentinc/confluent-kafka-go/kafka/offset.go generated vendored

@ -0,0 +1,145 @@
/**
* Copyright 2017 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package kafka
import (
"fmt"
"strconv"
)
/*
#include <stdlib.h>
#include "select_rdkafka.h"
static int64_t _c_rdkafka_offset_tail(int64_t rel) {
return RD_KAFKA_OFFSET_TAIL(rel);
}
*/
import "C"
// Offset type (int64) with support for canonical names
type Offset int64
// OffsetBeginning represents the earliest offset (logical)
const OffsetBeginning = Offset(C.RD_KAFKA_OFFSET_BEGINNING)
// OffsetEnd represents the latest offset (logical)
const OffsetEnd = Offset(C.RD_KAFKA_OFFSET_END)
// OffsetInvalid represents an invalid/unspecified offset
const OffsetInvalid = Offset(C.RD_KAFKA_OFFSET_INVALID)
// OffsetStored represents a stored offset
const OffsetStored = Offset(C.RD_KAFKA_OFFSET_STORED)
func (o Offset) String() string {
switch o {
case OffsetBeginning:
return "beginning"
case OffsetEnd:
return "end"
case OffsetInvalid:
return "unset"
case OffsetStored:
return "stored"
default:
return fmt.Sprintf("%d", int64(o))
}
}
// Set offset value, see NewOffset()
func (o *Offset) Set(offset interface{}) error {
n, err := NewOffset(offset)
if err == nil {
*o = n
}
return err
}
// NewOffset creates a new Offset using the provided logical string, or an
// absolute int64 offset value.
// Logical offsets: "beginning", "earliest", "end", "latest", "unset", "invalid", "stored"
func NewOffset(offset interface{}) (Offset, error) {
switch v := offset.(type) {
case string:
switch v {
case "beginning":
fallthrough
case "earliest":
return Offset(OffsetBeginning), nil
case "end":
fallthrough
case "latest":
return Offset(OffsetEnd), nil
case "unset":
fallthrough
case "invalid":
return Offset(OffsetInvalid), nil
case "stored":
return Offset(OffsetStored), nil
default:
off, err := strconv.Atoi(v)
return Offset(off), err
}
case int:
return Offset((int64)(v)), nil
case int64:
return Offset(v), nil
default:
return OffsetInvalid, newErrorFromString(ErrInvalidArg,
fmt.Sprintf("Invalid offset type: %t", v))
}
}
// OffsetTail returns the logical offset relativeOffset from current end of partition
func OffsetTail(relativeOffset Offset) Offset {
return Offset(C._c_rdkafka_offset_tail(C.int64_t(relativeOffset)))
}
// offsetsForTimes looks up offsets by timestamp for the given partitions.
//
// The returned offset for each partition is the earliest offset whose
// timestamp is greater than or equal to the given timestamp in the
// corresponding partition. If the provided timestamp exceeds that of the
// last message in the partition, a value of -1 will be returned.
//
// The timestamps to query are represented as `.Offset` in the `times`
// argument and the looked up offsets are represented as `.Offset` in the returned
// `offsets` list.
//
// The function will block for at most timeoutMs milliseconds.
//
// Duplicate Topic+Partitions are not supported.
// Per-partition errors may be returned in the `.Error` field.
func offsetsForTimes(H Handle, times []TopicPartition, timeoutMs int) (offsets []TopicPartition, err error) {
cparts := newCPartsFromTopicPartitions(times)
defer C.rd_kafka_topic_partition_list_destroy(cparts)
cerr := C.rd_kafka_offsets_for_times(H.gethandle().rk, cparts, C.int(timeoutMs))
if cerr != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return nil, newError(cerr)
}
return newTopicPartitionsFromCparts(cparts), nil
}

918
vendor/github.com/confluentinc/confluent-kafka-go/kafka/producer.go generated vendored

@ -0,0 +1,918 @@
/**
* Copyright 2016 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package kafka
import (
"context"
"fmt"
"math"
"time"
"unsafe"
)
/*
#include <stdlib.h>
#include "select_rdkafka.h"
#include "glue_rdkafka.h"
#ifdef RD_KAFKA_V_HEADERS
// Convert tmphdrs to chdrs (created by this function).
// If tmphdr.size == -1: value is considered Null
// tmphdr.size == 0: value is considered empty (ignored)
// tmphdr.size > 0: value is considered non-empty
//
// WARNING: The header keys and values will be freed by this function.
void tmphdrs_to_chdrs (tmphdr_t *tmphdrs, size_t tmphdrsCnt,
rd_kafka_headers_t **chdrs) {
size_t i;
*chdrs = rd_kafka_headers_new(tmphdrsCnt);
for (i = 0 ; i < tmphdrsCnt ; i++) {
rd_kafka_header_add(*chdrs,
tmphdrs[i].key, -1,
tmphdrs[i].size == -1 ? NULL :
(tmphdrs[i].size == 0 ? "" : tmphdrs[i].val),
tmphdrs[i].size == -1 ? 0 : tmphdrs[i].size);
if (tmphdrs[i].size > 0)
free((void *)tmphdrs[i].val);
free((void *)tmphdrs[i].key);
}
}
#else
void free_tmphdrs (tmphdr_t *tmphdrs, size_t tmphdrsCnt) {
size_t i;
for (i = 0 ; i < tmphdrsCnt ; i++) {
if (tmphdrs[i].size > 0)
free((void *)tmphdrs[i].val);
free((void *)tmphdrs[i].key);
}
}
#endif
rd_kafka_resp_err_t do_produce (rd_kafka_t *rk,
rd_kafka_topic_t *rkt, int32_t partition,
int msgflags,
int valIsNull, void *val, size_t val_len,
int keyIsNull, void *key, size_t key_len,
int64_t timestamp,
tmphdr_t *tmphdrs, size_t tmphdrsCnt,
uintptr_t cgoid) {
void *valp = valIsNull ? NULL : val;
void *keyp = keyIsNull ? NULL : key;
#ifdef RD_KAFKA_V_TIMESTAMP
rd_kafka_resp_err_t err;
#ifdef RD_KAFKA_V_HEADERS
rd_kafka_headers_t *hdrs = NULL;
#endif
#endif
if (tmphdrsCnt > 0) {
#ifdef RD_KAFKA_V_HEADERS
tmphdrs_to_chdrs(tmphdrs, tmphdrsCnt, &hdrs);
#else
free_tmphdrs(tmphdrs, tmphdrsCnt);
return RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED;
#endif
}
#ifdef RD_KAFKA_V_TIMESTAMP
err = rd_kafka_producev(rk,
RD_KAFKA_V_RKT(rkt),
RD_KAFKA_V_PARTITION(partition),
RD_KAFKA_V_MSGFLAGS(msgflags),
RD_KAFKA_V_VALUE(valp, val_len),
RD_KAFKA_V_KEY(keyp, key_len),
RD_KAFKA_V_TIMESTAMP(timestamp),
#ifdef RD_KAFKA_V_HEADERS
RD_KAFKA_V_HEADERS(hdrs),
#endif
RD_KAFKA_V_OPAQUE((void *)cgoid),
RD_KAFKA_V_END);
#ifdef RD_KAFKA_V_HEADERS
if (err && hdrs)
rd_kafka_headers_destroy(hdrs);
#endif
return err;
#else
if (timestamp)
return RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED;
if (rd_kafka_produce(rkt, partition, msgflags,
valp, val_len,
keyp, key_len,
(void *)cgoid) == -1)
return rd_kafka_last_error();
else
return RD_KAFKA_RESP_ERR_NO_ERROR;
#endif
}
*/
import "C"
// Producer implements a High-level Apache Kafka Producer instance
type Producer struct {
events chan Event
produceChannel chan *Message
handle handle
// Terminates the poller() goroutine
pollerTermChan chan bool
}
// String returns a human readable name for a Producer instance
func (p *Producer) String() string {
return p.handle.String()
}
// get_handle implements the Handle interface
func (p *Producer) gethandle() *handle {
return &p.handle
}
func (p *Producer) produce(msg *Message, msgFlags int, deliveryChan chan Event) error {
if msg == nil || msg.TopicPartition.Topic == nil || len(*msg.TopicPartition.Topic) == 0 {
return newErrorFromString(ErrInvalidArg, "")
}
crkt := p.handle.getRkt(*msg.TopicPartition.Topic)
// Three problems:
// 1) There's a difference between an empty Value or Key (length 0, proper pointer) and
// a null Value or Key (length 0, null pointer).
// 2) we need to be able to send a null Value or Key, but the unsafe.Pointer(&slice[0])
// dereference can't be performed on a nil slice.
// 3) cgo's pointer checking requires the unsafe.Pointer(slice..) call to be made
// in the call to the C function.
//
// Solution:
// Keep track of whether the Value or Key were nil (1), but let the valp and keyp pointers
// point to a 1-byte slice (but the length to send is still 0) so that the dereference (2)
// works.
// Then perform the unsafe.Pointer() on the valp and keyp pointers (which now either point
// to the original msg.Value and msg.Key or to the 1-byte slices) in the call to C (3).
//
var valp []byte
var keyp []byte
oneByte := []byte{0}
var valIsNull C.int
var keyIsNull C.int
var valLen int
var keyLen int
if msg.Value == nil {
valIsNull = 1
valLen = 0
valp = oneByte
} else {
valLen = len(msg.Value)
if valLen > 0 {
valp = msg.Value
} else {
valp = oneByte
}
}
if msg.Key == nil {
keyIsNull = 1
keyLen = 0
keyp = oneByte
} else {
keyLen = len(msg.Key)
if keyLen > 0 {
keyp = msg.Key
} else {
keyp = oneByte
}
}
var cgoid int
// Per-message state that needs to be retained through the C code:
// delivery channel (if specified)
// message opaque (if specified)
// Since these cant be passed as opaque pointers to the C code,
// due to cgo constraints, we add them to a per-producer map for lookup
// when the C code triggers the callbacks or events.
if deliveryChan != nil || msg.Opaque != nil {
cgoid = p.handle.cgoPut(cgoDr{deliveryChan: deliveryChan, opaque: msg.Opaque})
}
var timestamp int64
if !msg.Timestamp.IsZero() {
timestamp = msg.Timestamp.UnixNano() / 1000000
}
// Convert headers to C-friendly tmphdrs
var tmphdrs []C.tmphdr_t
tmphdrsCnt := len(msg.Headers)
if tmphdrsCnt > 0 {
tmphdrs = make([]C.tmphdr_t, tmphdrsCnt)
for n, hdr := range msg.Headers {
// Make a copy of the key
// to avoid runtime panic with
// foreign Go pointers in cgo.
tmphdrs[n].key = C.CString(hdr.Key)
if hdr.Value != nil {
tmphdrs[n].size = C.ssize_t(len(hdr.Value))
if tmphdrs[n].size > 0 {
// Make a copy of the value
// to avoid runtime panic with
// foreign Go pointers in cgo.
tmphdrs[n].val = C.CBytes(hdr.Value)
}
} else {
// null value
tmphdrs[n].size = C.ssize_t(-1)
}
}
} else {
// no headers, need a dummy tmphdrs of size 1 to avoid index
// out of bounds panic in do_produce() call below.
// tmphdrsCnt will be 0.
tmphdrs = []C.tmphdr_t{{nil, nil, 0}}
}
cErr := C.do_produce(p.handle.rk, crkt,
C.int32_t(msg.TopicPartition.Partition),
C.int(msgFlags)|C.RD_KAFKA_MSG_F_COPY,
valIsNull, unsafe.Pointer(&valp[0]), C.size_t(valLen),
keyIsNull, unsafe.Pointer(&keyp[0]), C.size_t(keyLen),
C.int64_t(timestamp),
(*C.tmphdr_t)(unsafe.Pointer(&tmphdrs[0])), C.size_t(tmphdrsCnt),
(C.uintptr_t)(cgoid))
if cErr != C.RD_KAFKA_RESP_ERR_NO_ERROR {
if cgoid != 0 {
p.handle.cgoGet(cgoid)
}
return newError(cErr)
}
return nil
}
// Produce single message.
// This is an asynchronous call that enqueues the message on the internal
// transmit queue, thus returning immediately.
// The delivery report will be sent on the provided deliveryChan if specified,
// or on the Producer object's Events() channel if not.
// msg.Timestamp requires librdkafka >= 0.9.4 (else returns ErrNotImplemented),
// api.version.request=true, and broker >= 0.10.0.0.
// msg.Headers requires librdkafka >= 0.11.4 (else returns ErrNotImplemented),
// api.version.request=true, and broker >= 0.11.0.0.
// Returns an error if message could not be enqueued.
func (p *Producer) Produce(msg *Message, deliveryChan chan Event) error {
return p.produce(msg, 0, deliveryChan)
}
// Produce a batch of messages.
// These batches do not relate to the message batches sent to the broker, the latter
// are collected on the fly internally in librdkafka.
// WARNING: This is an experimental API.
// NOTE: timestamps and headers are not supported with this API.
func (p *Producer) produceBatch(topic string, msgs []*Message, msgFlags int) error {
crkt := p.handle.getRkt(topic)
cmsgs := make([]C.rd_kafka_message_t, len(msgs))
for i, m := range msgs {
p.handle.messageToC(m, &cmsgs[i])
}
r := C.rd_kafka_produce_batch(crkt, C.RD_KAFKA_PARTITION_UA, C.int(msgFlags)|C.RD_KAFKA_MSG_F_FREE,
(*C.rd_kafka_message_t)(&cmsgs[0]), C.int(len(msgs)))
if r == -1 {
return newError(C.rd_kafka_last_error())
}
return nil
}
// Events returns the Events channel (read)
func (p *Producer) Events() chan Event {
return p.events
}
// Logs returns the Log channel (if enabled), else nil
func (p *Producer) Logs() chan LogEvent {
return p.handle.logs
}
// ProduceChannel returns the produce *Message channel (write)
func (p *Producer) ProduceChannel() chan *Message {
return p.produceChannel
}
// Len returns the number of messages and requests waiting to be transmitted to the broker
// as well as delivery reports queued for the application.
// Includes messages on ProduceChannel.
func (p *Producer) Len() int {
return len(p.produceChannel) + len(p.events) + int(C.rd_kafka_outq_len(p.handle.rk))
}
// Flush and wait for outstanding messages and requests to complete delivery.
// Includes messages on ProduceChannel.
// Runs until value reaches zero or on timeoutMs.
// Returns the number of outstanding events still un-flushed.
func (p *Producer) Flush(timeoutMs int) int {
termChan := make(chan bool) // unused stand-in termChan
d, _ := time.ParseDuration(fmt.Sprintf("%dms", timeoutMs))
tEnd := time.Now().Add(d)
for p.Len() > 0 {
remain := tEnd.Sub(time.Now()).Seconds()
if remain <= 0.0 {
return p.Len()
}
p.handle.eventPoll(p.events,
int(math.Min(100, remain*1000)), 1000, termChan)
}
return 0
}
// Close a Producer instance.
// The Producer object or its channels are no longer usable after this call.
func (p *Producer) Close() {
// Wait for poller() (signaled by closing pollerTermChan)
// and channel_producer() (signaled by closing ProduceChannel)
close(p.pollerTermChan)
close(p.produceChannel)
p.handle.waitGroup.Wait()
close(p.events)
p.handle.cleanup()
C.rd_kafka_destroy(p.handle.rk)
}
const (
// PurgeInFlight purges messages in-flight to or from the broker.
// Purging these messages will void any future acknowledgements from the
// broker, making it impossible for the application to know if these
// messages were successfully delivered or not.
// Retrying these messages may lead to duplicates.
PurgeInFlight = int(C.RD_KAFKA_PURGE_F_INFLIGHT)
// PurgeQueue Purge messages in internal queues.
PurgeQueue = int(C.RD_KAFKA_PURGE_F_QUEUE)
// PurgeNonBlocking Don't wait for background thread queue purging to finish.
PurgeNonBlocking = int(C.RD_KAFKA_PURGE_F_NON_BLOCKING)
)
// Purge messages currently handled by this producer instance.
//
// flags is a combination of PurgeQueue, PurgeInFlight and PurgeNonBlocking.
//
// The application will need to call Poll(), Flush() or read the Events() channel
// after this call to serve delivery reports for the purged messages.
//
// Messages purged from internal queues fail with the delivery report
// error code set to ErrPurgeQueue, while purged messages that
// are in-flight to or from the broker will fail with the error code set to
// ErrPurgeInflight.
//
// Warning: Purging messages that are in-flight to or from the broker
// will ignore any sub-sequent acknowledgement for these messages
// received from the broker, effectively making it impossible
// for the application to know if the messages were successfully
// produced or not. This may result in duplicate messages if the
// application retries these messages at a later time.
//
// Note: This call may block for a short time while background thread
// queues are purged.
//
// Returns nil on success, ErrInvalidArg if the purge flags are invalid or unknown.
func (p *Producer) Purge(flags int) error {
cErr := C.rd_kafka_purge(p.handle.rk, C.int(flags))
if cErr != C.RD_KAFKA_RESP_ERR_NO_ERROR {
return newError(cErr)
}
return nil
}
// NewProducer creates a new high-level Producer instance.
//
// conf is a *ConfigMap with standard librdkafka configuration properties.
//
// Supported special configuration properties (type, default):
// go.batch.producer (bool, false) - EXPERIMENTAL: Enable batch producer (for increased performance).
// These batches do not relate to Kafka message batches in any way.
// Note: timestamps and headers are not supported with this interface.
// go.delivery.reports (bool, true) - Forward per-message delivery reports to the
// Events() channel.
// go.delivery.report.fields (string, "key,value") - Comma separated list of fields to enable for delivery reports.
// Allowed values: all, none (or empty string), key, value, headers
// Warning: There is a performance penalty to include headers in the delivery report.
// go.events.channel.size (int, 1000000) - Events().
// go.produce.channel.size (int, 1000000) - ProduceChannel() buffer size (in number of messages)
// go.logs.channel.enable (bool, false) - Forward log to Logs() channel.
// go.logs.channel (chan kafka.LogEvent, nil) - Forward logs to application-provided channel instead of Logs(). Requires go.logs.channel.enable=true.
//
func NewProducer(conf *ConfigMap) (*Producer, error) {
err := versionCheck()
if err != nil {
return nil, err
}
p := &Producer{}
// before we do anything with the configuration, create a copy such that
// the original is not mutated.
confCopy := conf.clone()
v, err := confCopy.extract("delivery.report.only.error", false)
if v == true {
// FIXME: The filtering of successful DRs must be done in
// the Go client to avoid cgoDr memory leaks.
return nil, newErrorFromString(ErrUnsupportedFeature,
"delivery.report.only.error=true is not currently supported by the Go client")
}
v, err = confCopy.extract("go.batch.producer", false)
if err != nil {
return nil, err
}
batchProducer := v.(bool)
v, err = confCopy.extract("go.delivery.reports", true)
if err != nil {
return nil, err
}
p.handle.fwdDr = v.(bool)
v, err = confCopy.extract("go.delivery.report.fields", "key,value")
if err != nil {
return nil, err
}
p.handle.msgFields, err = newMessageFieldsFrom(v)
if err != nil {
return nil, err
}
v, err = confCopy.extract("go.events.channel.size", 1000000)
if err != nil {
return nil, err
}
eventsChanSize := v.(int)
v, err = confCopy.extract("go.produce.channel.size", 1000000)
if err != nil {
return nil, err
}
produceChannelSize := v.(int)
logsChanEnable, logsChan, err := confCopy.extractLogConfig()
if err != nil {
return nil, err
}
if int(C.rd_kafka_version()) < 0x01000000 {
// produce.offset.report is no longer used in librdkafka >= v1.0.0
v, _ = confCopy.extract("{topic}.produce.offset.report", nil)
if v == nil {
// Enable offset reporting by default, unless overriden.
confCopy.SetKey("{topic}.produce.offset.report", true)
}
}
// Convert ConfigMap to librdkafka conf_t
cConf, err := confCopy.convert()
if err != nil {
return nil, err
}
cErrstr := (*C.char)(C.malloc(C.size_t(256)))
defer C.free(unsafe.Pointer(cErrstr))
C.rd_kafka_conf_set_events(cConf, C.RD_KAFKA_EVENT_DR|C.RD_KAFKA_EVENT_STATS|C.RD_KAFKA_EVENT_ERROR|C.RD_KAFKA_EVENT_OAUTHBEARER_TOKEN_REFRESH)
// Create librdkafka producer instance
p.handle.rk = C.rd_kafka_new(C.RD_KAFKA_PRODUCER, cConf, cErrstr, 256)
if p.handle.rk == nil {
return nil, newErrorFromCString(C.RD_KAFKA_RESP_ERR__INVALID_ARG, cErrstr)
}
p.handle.p = p
p.handle.setup()
p.handle.rkq = C.rd_kafka_queue_get_main(p.handle.rk)
p.events = make(chan Event, eventsChanSize)
p.produceChannel = make(chan *Message, produceChannelSize)
p.pollerTermChan = make(chan bool)
if logsChanEnable {
p.handle.setupLogQueue(logsChan, p.pollerTermChan)
}
p.handle.waitGroup.Add(1)
go func() {
poller(p, p.pollerTermChan)
p.handle.waitGroup.Done()
}()
// non-batch or batch producer, only one must be used
var producer func(*Producer)
if batchProducer {
producer = channelBatchProducer
} else {
producer = channelProducer
}
p.handle.waitGroup.Add(1)
go func() {
producer(p)
p.handle.waitGroup.Done()
}()
return p, nil
}
// channel_producer serves the ProduceChannel channel
func channelProducer(p *Producer) {
for m := range p.produceChannel {
err := p.produce(m, C.RD_KAFKA_MSG_F_BLOCK, nil)
if err != nil {
m.TopicPartition.Error = err
p.events <- m
}
}
}
// channelBatchProducer serves the ProduceChannel channel and attempts to
// improve cgo performance by using the produceBatch() interface.
func channelBatchProducer(p *Producer) {
var buffered = make(map[string][]*Message)
bufferedCnt := 0
const batchSize int = 1000000
totMsgCnt := 0
totBatchCnt := 0
for m := range p.produceChannel {
buffered[*m.TopicPartition.Topic] = append(buffered[*m.TopicPartition.Topic], m)
bufferedCnt++
loop2:
for true {
select {
case m, ok := <-p.produceChannel:
if !ok {
break loop2
}
if m == nil {
panic("nil message received on ProduceChannel")
}
if m.TopicPartition.Topic == nil {
panic(fmt.Sprintf("message without Topic received on ProduceChannel: %v", m))
}
buffered[*m.TopicPartition.Topic] = append(buffered[*m.TopicPartition.Topic], m)
bufferedCnt++
if bufferedCnt >= batchSize {
break loop2
}
default:
break loop2
}
}
totBatchCnt++
totMsgCnt += len(buffered)
for topic, buffered2 := range buffered {
err := p.produceBatch(topic, buffered2, C.RD_KAFKA_MSG_F_BLOCK)
if err != nil {
for _, m = range buffered2 {
m.TopicPartition.Error = err
p.events <- m
}
}
}
buffered = make(map[string][]*Message)
bufferedCnt = 0
}
}
// poller polls the rd_kafka_t handle for events until signalled for termination
func poller(p *Producer, termChan chan bool) {
for {
select {
case _ = <-termChan:
return
default:
_, term := p.handle.eventPoll(p.events, 100, 1000, termChan)
if term {
return
}
break
}
}
}
// GetMetadata queries broker for cluster and topic metadata.
// If topic is non-nil only information about that topic is returned, else if
// allTopics is false only information about locally used topics is returned,
// else information about all topics is returned.
// GetMetadata is equivalent to listTopics, describeTopics and describeCluster in the Java API.
func (p *Producer) GetMetadata(topic *string, allTopics bool, timeoutMs int) (*Metadata, error) {
return getMetadata(p, topic, allTopics, timeoutMs)
}
// QueryWatermarkOffsets returns the broker's low and high offsets for the given topic
// and partition.
func (p *Producer) QueryWatermarkOffsets(topic string, partition int32, timeoutMs int) (low, high int64, err error) {
return queryWatermarkOffsets(p, topic, partition, timeoutMs)
}
// OffsetsForTimes looks up offsets by timestamp for the given partitions.
//
// The returned offset for each partition is the earliest offset whose
// timestamp is greater than or equal to the given timestamp in the
// corresponding partition. If the provided timestamp exceeds that of the
// last message in the partition, a value of -1 will be returned.
//
// The timestamps to query are represented as `.Offset` in the `times`
// argument and the looked up offsets are represented as `.Offset` in the returned
// `offsets` list.
//
// The function will block for at most timeoutMs milliseconds.
//
// Duplicate Topic+Partitions are not supported.
// Per-partition errors may be returned in the `.Error` field.
func (p *Producer) OffsetsForTimes(times []TopicPartition, timeoutMs int) (offsets []TopicPartition, err error) {
return offsetsForTimes(p, times, timeoutMs)
}
// GetFatalError returns an Error object if the client instance has raised a fatal error, else nil.
func (p *Producer) GetFatalError() error {
return getFatalError(p)
}
// TestFatalError triggers a fatal error in the underlying client.
// This is to be used strictly for testing purposes.
func (p *Producer) TestFatalError(code ErrorCode, str string) ErrorCode {
return testFatalError(p, code, str)
}
// SetOAuthBearerToken sets the the data to be transmitted
// to a broker during SASL/OAUTHBEARER authentication. It will return nil
// on success, otherwise an error if:
// 1) the token data is invalid (meaning an expiration time in the past
// or either a token value or an extension key or value that does not meet
// the regular expression requirements as per
// https://tools.ietf.org/html/rfc7628#section-3.1);
// 2) SASL/OAUTHBEARER is not supported by the underlying librdkafka build;
// 3) SASL/OAUTHBEARER is supported but is not configured as the client's
// authentication mechanism.
func (p *Producer) SetOAuthBearerToken(oauthBearerToken OAuthBearerToken) error {
return p.handle.setOAuthBearerToken(oauthBearerToken)
}
// SetOAuthBearerTokenFailure sets the error message describing why token
// retrieval/setting failed; it also schedules a new token refresh event for 10
// seconds later so the attempt may be retried. It will return nil on
// success, otherwise an error if:
// 1) SASL/OAUTHBEARER is not supported by the underlying librdkafka build;
// 2) SASL/OAUTHBEARER is supported but is not configured as the client's
// authentication mechanism.
func (p *Producer) SetOAuthBearerTokenFailure(errstr string) error {
return p.handle.setOAuthBearerTokenFailure(errstr)
}
// Transactional API
// InitTransactions Initializes transactions for the producer instance.
//
// This function ensures any transactions initiated by previous instances
// of the producer with the same `transactional.id` are completed.
// If the previous instance failed with a transaction in progress the
// previous transaction will be aborted.
// This function needs to be called before any other transactional or
// produce functions are called when the `transactional.id` is configured.
//
// If the last transaction had begun completion (following transaction commit)
// but not yet finished, this function will await the previous transaction's
// completion.
//
// When any previous transactions have been fenced this function
// will acquire the internal producer id and epoch, used in all future
// transactional messages issued by this producer instance.
//
// Upon successful return from this function the application has to perform at
// least one of the following operations within `transaction.timeout.ms` to
// avoid timing out the transaction on the broker:
// * `Produce()` (et.al)
// * `SendOffsetsToTransaction()`
// * `CommitTransaction()`
// * `AbortTransaction()`
//
// Parameters:
// * `ctx` - The maximum time to block, or nil for indefinite.
// On timeout the operation may continue in the background,
// depending on state, and it is okay to call `InitTransactions()`
// again.
//
// Returns nil on success or an error on failure.
// Check whether the returned error object permits retrying
// by calling `err.(kafka.Error).IsRetriable()`, or whether a fatal
// error has been raised by calling `err.(kafka.Error).IsFatal()`.
func (p *Producer) InitTransactions(ctx context.Context) error {
cError := C.rd_kafka_init_transactions(p.handle.rk,
cTimeoutFromContext(ctx))
if cError != nil {
return newErrorFromCErrorDestroy(cError)
}
return nil
}
// BeginTransaction starts a new transaction.
//
// `InitTransactions()` must have been called successfully (once)
// before this function is called.
//
// Any messages produced, offsets sent (`SendOffsetsToTransaction()`),
// etc, after the successful return of this function will be part of
// the transaction and committed or aborted atomatically.
//
// Finish the transaction by calling `CommitTransaction()` or
// abort the transaction by calling `AbortTransaction()`.
//
// Returns nil on success or an error object on failure.
// Check whether a fatal error has been raised by
// calling `err.(kafka.Error).IsFatal()`.
//
// Note: With the transactional producer, `Produce()`, et.al, are only
// allowed during an on-going transaction, as started with this function.
// Any produce call outside an on-going transaction, or for a failed
// transaction, will fail.
func (p *Producer) BeginTransaction() error {
cError := C.rd_kafka_begin_transaction(p.handle.rk)
if cError != nil {
return newErrorFromCErrorDestroy(cError)
}
return nil
}
// SendOffsetsToTransaction sends a list of topic partition offsets to the
// consumer group coordinator for `consumerMetadata`, and marks the offsets
// as part part of the current transaction.
// These offsets will be considered committed only if the transaction is
// committed successfully.
//
// The offsets should be the next message your application will consume,
// i.e., the last processed message's offset + 1 for each partition.
// Either track the offsets manually during processing or use
// `consumer.Position()` (on the consumer) to get the current offsets for
// the partitions assigned to the consumer.
//
// Use this method at the end of a consume-transform-produce loop prior
// to committing the transaction with `CommitTransaction()`.
//
// Parameters:
// * `ctx` - The maximum amount of time to block, or nil for indefinite.
// * `offsets` - List of offsets to commit to the consumer group upon
// successful commit of the transaction. Offsets should be
// the next message to consume, e.g., last processed message + 1.
// * `consumerMetadata` - The current consumer group metadata as returned by
// `consumer.GetConsumerGroupMetadata()` on the consumer
// instance the provided offsets were consumed from.
//
// Note: The consumer must disable auto commits (set `enable.auto.commit` to false on the consumer).
//
// Note: Logical and invalid offsets (e.g., OffsetInvalid) in
// `offsets` will be ignored. If there are no valid offsets in
// `offsets` the function will return nil and no action will be taken.
//
// Returns nil on success or an error object on failure.
// Check whether the returned error object permits retrying
// by calling `err.(kafka.Error).IsRetriable()`, or whether an abortable
// or fatal error has been raised by calling
// `err.(kafka.Error).TxnRequiresAbort()` or `err.(kafka.Error).IsFatal()`
// respectively.
func (p *Producer) SendOffsetsToTransaction(ctx context.Context, offsets []TopicPartition, consumerMetadata *ConsumerGroupMetadata) error {
var cOffsets *C.rd_kafka_topic_partition_list_t
if offsets != nil {
cOffsets = newCPartsFromTopicPartitions(offsets)
defer C.rd_kafka_topic_partition_list_destroy(cOffsets)
}
cgmd, err := deserializeConsumerGroupMetadata(consumerMetadata.serialized)
if err != nil {
return err
}
defer C.rd_kafka_consumer_group_metadata_destroy(cgmd)
cError := C.rd_kafka_send_offsets_to_transaction(
p.handle.rk,
cOffsets,
cgmd,
cTimeoutFromContext(ctx))
if cError != nil {
return newErrorFromCErrorDestroy(cError)
}
return nil
}
// CommitTransaction commits the current transaction.
//
// Any outstanding messages will be flushed (delivered) before actually
// committing the transaction.
//
// If any of the outstanding messages fail permanently the current
// transaction will enter the abortable error state and this
// function will return an abortable error, in this case the application
// must call `AbortTransaction()` before attempting a new
// transaction with `BeginTransaction()`.
//
// Parameters:
// * `ctx` - The maximum amount of time to block, or nil for indefinite.
//
// Note: This function will block until all outstanding messages are
// delivered and the transaction commit request has been successfully
// handled by the transaction coordinator, or until the `ctx` expires,
// which ever comes first. On timeout the application may
// call the function again.
//
// Note: Will automatically call `Flush()` to ensure all queued
// messages are delivered before attempting to commit the transaction.
// The application MUST serve the `producer.Events()` channel for delivery
// reports in a separate go-routine during this time.
//
// Returns nil on success or an error object on failure.
// Check whether the returned error object permits retrying
// by calling `err.(kafka.Error).IsRetriable()`, or whether an abortable
// or fatal error has been raised by calling
// `err.(kafka.Error).TxnRequiresAbort()` or `err.(kafka.Error).IsFatal()`
// respectively.
func (p *Producer) CommitTransaction(ctx context.Context) error {
cError := C.rd_kafka_commit_transaction(p.handle.rk,
cTimeoutFromContext(ctx))
if cError != nil {
return newErrorFromCErrorDestroy(cError)
}
return nil
}
// AbortTransaction aborts the ongoing transaction.
//
// This function should also be used to recover from non-fatal abortable
// transaction errors.
//
// Any outstanding messages will be purged and fail with
// `ErrPurgeInflight` or `ErrPurgeQueue`.
//
// Parameters:
// * `ctx` - The maximum amount of time to block, or nil for indefinite.
//
// Note: This function will block until all outstanding messages are purged
// and the transaction abort request has been successfully
// handled by the transaction coordinator, or until the `ctx` expires,
// which ever comes first. On timeout the application may
// call the function again.
//
// Note: Will automatically call `Purge()` and `Flush()` to ensure all queued
// and in-flight messages are purged before attempting to abort the transaction.
// The application MUST serve the `producer.Events()` channel for delivery
// reports in a separate go-routine during this time.
//
// Returns nil on success or an error object on failure.
// Check whether the returned error object permits retrying
// by calling `err.(kafka.Error).IsRetriable()`, or whether a fatal error
// has been raised by calling `err.(kafka.Error).IsFatal()`.
func (p *Producer) AbortTransaction(ctx context.Context) error {
cError := C.rd_kafka_abort_transaction(p.handle.rk,
cTimeoutFromContext(ctx))
if cError != nil {
return newErrorFromCErrorDestroy(cError)
}
return nil
}

29
vendor/github.com/confluentinc/confluent-kafka-go/kafka/select_rdkafka.h generated vendored

@ -0,0 +1,29 @@
/**
* Copyright 2020 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This file uses a preprocessor macro defined by the various build_*.go
// files to determine whether to import the bundled librdkafka header, or
// the system one.
// This is needed because cgo will automatically add -I. to the include
// path, so <librdkafka/rdkafka.h> would find a bundled header instead of
// the system one if it were called librdkafka/rdkafka.h instead of
// librdkafka_vendor/rdkafka.h
#ifdef USE_VENDORED_LIBRDKAFKA
#include "librdkafka_vendor/rdkafka.h"
#else
#include <librdkafka/rdkafka.h>
#endif

8
vendor/github.com/confluentinc/confluent-kafka-go/kafka/testconf-example.json generated vendored

@ -0,0 +1,8 @@
{
"Brokers": "mybroker or $BROKERS env",
"Topic": "test",
"GroupID": "testgroup",
"PerfMsgCount": 1000000,
"PerfMsgSize": 100,
"Config": ["api.version.request=true"]
}

248
vendor/github.com/confluentinc/confluent-kafka-go/kafka/testhelpers.go generated vendored

@ -0,0 +1,248 @@
package kafka
/**
* Copyright 2016 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import (
"context"
"encoding/json"
"fmt"
"math/rand"
"os"
"testing"
"time"
)
/*
#include "select_rdkafka.h"
*/
import "C"
var testconf struct {
Brokers string
Topic string
GroupID string
PerfMsgCount int
PerfMsgSize int
Config []string
conf ConfigMap
}
// testconf_read reads the test suite config file testconf.json which must
// contain at least Brokers and Topic string properties.
// Returns true if the testconf was found and usable, false if no such file, or panics
// if the file format is wrong.
func testconfRead() bool {
cf, err := os.Open("testconf.json")
if err != nil {
fmt.Fprintf(os.Stderr, "%% testconf.json not found - ignoring test\n")
return false
}
// Default values
testconf.PerfMsgCount = 2000000
testconf.PerfMsgSize = 100
testconf.GroupID = "testgroup"
jp := json.NewDecoder(cf)
err = jp.Decode(&testconf)
if err != nil {
panic(fmt.Sprintf("Failed to parse testconf: %s", err))
}
cf.Close()
if testconf.Brokers[0] == '$' {
// Read broker list from environment variable
testconf.Brokers = os.Getenv(testconf.Brokers[1:])
}
if testconf.Brokers == "" || testconf.Topic == "" {
panic("Missing Brokers or Topic in testconf.json")
}
return true
}
// update existing ConfigMap with key=value pairs from testconf.Config
func (cm *ConfigMap) updateFromTestconf() error {
if testconf.Config == nil {
return nil
}
// Translate "key=value" pairs in Config to ConfigMap
for _, s := range testconf.Config {
err := cm.Set(s)
if err != nil {
return err
}
}
return nil
}
// Return the number of messages available in all partitions of a topic.
// WARNING: This uses watermark offsets so it will be incorrect for compacted topics.
func getMessageCountInTopic(topic string) (int, error) {
// Create consumer
config := &ConfigMap{"bootstrap.servers": testconf.Brokers,
"group.id": testconf.GroupID}
config.updateFromTestconf()
c, err := NewConsumer(config)
if err != nil {
return 0, err
}
defer c.Close()
// get metadata for the topic to find out number of partitions
metadata, err := c.GetMetadata(&topic, false, 5*1000)
if err != nil {
return 0, err
}
t, ok := metadata.Topics[topic]
if !ok {
return 0, newError(C.RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC)
}
cnt := 0
for _, p := range t.Partitions {
low, high, err := c.QueryWatermarkOffsets(topic, p.ID, 5*1000)
if err != nil {
continue
}
cnt += int(high - low)
}
return cnt, nil
}
// getBrokerList returns a list of brokers (ids) in the cluster
func getBrokerList(H Handle) (brokers []int32, err error) {
md, err := getMetadata(H, nil, true, 15*1000)
if err != nil {
return nil, err
}
brokers = make([]int32, len(md.Brokers))
for i, mdBroker := range md.Brokers {
brokers[i] = mdBroker.ID
}
return brokers, nil
}
// waitTopicInMetadata waits for the given topic to show up in metadata
func waitTopicInMetadata(H Handle, topic string, timeoutMs int) error {
d, _ := time.ParseDuration(fmt.Sprintf("%dms", timeoutMs))
tEnd := time.Now().Add(d)
for {
remain := tEnd.Sub(time.Now()).Seconds()
if remain < 0.0 {
return newErrorFromString(ErrTimedOut,
fmt.Sprintf("Timed out waiting for topic %s to appear in metadata", topic))
}
md, err := getMetadata(H, nil, true, int(remain*1000))
if err != nil {
return err
}
for _, t := range md.Topics {
if t.Topic != topic {
continue
}
if t.Error.Code() != ErrNoError || len(t.Partitions) < 1 {
continue
}
// Proper topic found in metadata
return nil
}
time.Sleep(500 * 1000) // 500ms
}
}
func createAdminClient(t *testing.T) (a *AdminClient) {
numver, strver := LibraryVersion()
if numver < 0x000b0500 {
t.Skipf("Requires librdkafka >=0.11.5 (currently on %s, 0x%x)", strver, numver)
}
if !testconfRead() {
t.Skipf("Missing testconf.json")
}
conf := ConfigMap{"bootstrap.servers": testconf.Brokers}
conf.updateFromTestconf()
/*
* Create producer and produce a couple of messages with and without
* headers.
*/
a, err := NewAdminClient(&conf)
if err != nil {
t.Fatalf("NewAdminClient: %v", err)
}
return a
}
func createTestTopic(t *testing.T, suffix string, numPartitions int, replicationFactor int) string {
rand.Seed(time.Now().Unix())
topic := fmt.Sprintf("%s-%s-%d", testconf.Topic, suffix, rand.Intn(100000))
a := createAdminClient(t)
defer a.Close()
newTopics := []TopicSpecification{
{
Topic: topic,
NumPartitions: numPartitions,
ReplicationFactor: replicationFactor,
},
}
maxDuration, err := time.ParseDuration("30s")
if err != nil {
t.Fatalf("%s", err)
}
ctx, cancel := context.WithTimeout(context.Background(), maxDuration)
defer cancel()
result, err := a.CreateTopics(ctx, newTopics, nil)
if err != nil {
t.Fatalf("CreateTopics() failed: %s", err)
}
for _, res := range result {
if res.Error.Code() != ErrNoError {
t.Errorf("Failed to create topic %s: %s\n",
res.Topic, res.Error)
continue
}
}
return topic
}

55
vendor/github.com/confluentinc/confluent-kafka-go/kafka/time.go generated vendored

@ -0,0 +1,55 @@
/**
* Copyright 2019 Confluent Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package kafka
import "C"
import (
"context"
"time"
)
const (
cTimeoutInfinite = C.int(-1) // Blocks indefinitely until completion.
cTimeoutNoWait = C.int(0) // Returns immediately without blocking.
)
// cTimeoutFromContext returns the remaining time after which work done on behalf of this context
// should be canceled, in milliseconds.
//
// If no deadline/timeout is set, or if the timeout does not fit in an int32, it returns
// cTimeoutInfinite;
// If there is no time left in this context, it returns cTimeoutNoWait.
func cTimeoutFromContext(ctx context.Context) C.int {
if ctx == nil {
return cTimeoutInfinite
}
timeout, hasTimeout := timeout(ctx)
if !hasTimeout {
return cTimeoutInfinite
}
if timeout <= 0 {
return cTimeoutNoWait
}
timeoutMs := int64(timeout / time.Millisecond)
if int64(int32(timeoutMs)) < timeoutMs {
return cTimeoutInfinite
}
return C.int(timeoutMs)
}

21
vendor/github.com/go-stack/stack/LICENSE.md generated vendored

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2014 Chris Hines
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

38
vendor/github.com/go-stack/stack/README.md generated vendored

@ -0,0 +1,38 @@
[![GoDoc](https://godoc.org/github.com/go-stack/stack?status.svg)](https://godoc.org/github.com/go-stack/stack)
[![Go Report Card](https://goreportcard.com/badge/go-stack/stack)](https://goreportcard.com/report/go-stack/stack)
[![TravisCI](https://travis-ci.org/go-stack/stack.svg?branch=master)](https://travis-ci.org/go-stack/stack)
[![Coverage Status](https://coveralls.io/repos/github/go-stack/stack/badge.svg?branch=master)](https://coveralls.io/github/go-stack/stack?branch=master)
# stack
Package stack implements utilities to capture, manipulate, and format call
stacks. It provides a simpler API than package runtime.
The implementation takes care of the minutia and special cases of interpreting
the program counter (pc) values returned by runtime.Callers.
## Versioning
Package stack publishes releases via [semver](http://semver.org/) compatible Git
tags prefixed with a single 'v'. The master branch always contains the latest
release. The develop branch contains unreleased commits.
## Formatting
Package stack's types implement fmt.Formatter, which provides a simple and
flexible way to declaratively configure formatting when used with logging or
error tracking packages.
```go
func DoTheThing() {
c := stack.Caller(0)
log.Print(c) // "source.go:10"
log.Printf("%+v", c) // "pkg/path/source.go:10"
log.Printf("%n", c) // "DoTheThing"
s := stack.Trace().TrimRuntime()
log.Print(s) // "[source.go:15 caller.go:42 main.go:14]"
}
```
See the docs for all of the supported formatting options.

400
vendor/github.com/go-stack/stack/stack.go generated vendored

@ -0,0 +1,400 @@
// +build go1.7
// Package stack implements utilities to capture, manipulate, and format call
// stacks. It provides a simpler API than package runtime.
//
// The implementation takes care of the minutia and special cases of
// interpreting the program counter (pc) values returned by runtime.Callers.
//
// Package stack's types implement fmt.Formatter, which provides a simple and
// flexible way to declaratively configure formatting when used with logging
// or error tracking packages.
package stack
import (
"bytes"
"errors"
"fmt"
"io"
"runtime"
"strconv"
"strings"
)
// Call records a single function invocation from a goroutine stack.
type Call struct {
frame runtime.Frame
}
// Caller returns a Call from the stack of the current goroutine. The argument
// skip is the number of stack frames to ascend, with 0 identifying the
// calling function.
func Caller(skip int) Call {
// As of Go 1.9 we need room for up to three PC entries.
//
// 0. An entry for the stack frame prior to the target to check for
// special handling needed if that prior entry is runtime.sigpanic.
// 1. A possible second entry to hold metadata about skipped inlined
// functions. If inline functions were not skipped the target frame
// PC will be here.
// 2. A third entry for the target frame PC when the second entry
// is used for skipped inline functions.
var pcs [3]uintptr
n := runtime.Callers(skip+1, pcs[:])
frames := runtime.CallersFrames(pcs[:n])
frame, _ := frames.Next()
frame, _ = frames.Next()
return Call{
frame: frame,
}
}
// String implements fmt.Stinger. It is equivalent to fmt.Sprintf("%v", c).
func (c Call) String() string {
return fmt.Sprint(c)
}
// MarshalText implements encoding.TextMarshaler. It formats the Call the same
// as fmt.Sprintf("%v", c).
func (c Call) MarshalText() ([]byte, error) {
if c.frame == (runtime.Frame{}) {
return nil, ErrNoFunc
}
buf := bytes.Buffer{}
fmt.Fprint(&buf, c)
return buf.Bytes(), nil
}
// ErrNoFunc means that the Call has a nil *runtime.Func. The most likely
// cause is a Call with the zero value.
var ErrNoFunc = errors.New("no call stack information")
// Format implements fmt.Formatter with support for the following verbs.
//
// %s source file
// %d line number
// %n function name
// %k last segment of the package path
// %v equivalent to %s:%d
//
// It accepts the '+' and '#' flags for most of the verbs as follows.
//
// %+s path of source file relative to the compile time GOPATH,
// or the module path joined to the path of source file relative
// to module root
// %#s full path of source file
// %+n import path qualified function name
// %+k full package path
// %+v equivalent to %+s:%d
// %#v equivalent to %#s:%d
func (c Call) Format(s fmt.State, verb rune) {
if c.frame == (runtime.Frame{}) {
fmt.Fprintf(s, "%%!%c(NOFUNC)", verb)
return
}
switch verb {
case 's', 'v':
file := c.frame.File
switch {
case s.Flag('#'):
// done
case s.Flag('+'):
file = pkgFilePath(&c.frame)
default:
const sep = "/"
if i := strings.LastIndex(file, sep); i != -1 {
file = file[i+len(sep):]
}
}
io.WriteString(s, file)
if verb == 'v' {
buf := [7]byte{':'}
s.Write(strconv.AppendInt(buf[:1], int64(c.frame.Line), 10))
}
case 'd':
buf := [6]byte{}
s.Write(strconv.AppendInt(buf[:0], int64(c.frame.Line), 10))
case 'k':
name := c.frame.Function
const pathSep = "/"
start, end := 0, len(name)
if i := strings.LastIndex(name, pathSep); i != -1 {
start = i + len(pathSep)
}
const pkgSep = "."
if i := strings.Index(name[start:], pkgSep); i != -1 {
end = start + i
}
if s.Flag('+') {
start = 0
}
io.WriteString(s, name[start:end])
case 'n':
name := c.frame.Function
if !s.Flag('+') {
const pathSep = "/"
if i := strings.LastIndex(name, pathSep); i != -1 {
name = name[i+len(pathSep):]
}
const pkgSep = "."
if i := strings.Index(name, pkgSep); i != -1 {
name = name[i+len(pkgSep):]
}
}
io.WriteString(s, name)
}
}
// Frame returns the call frame infomation for the Call.
func (c Call) Frame() runtime.Frame {
return c.frame
}
// PC returns the program counter for this call frame; multiple frames may
// have the same PC value.
//
// Deprecated: Use Call.Frame instead.
func (c Call) PC() uintptr {
return c.frame.PC
}
// CallStack records a sequence of function invocations from a goroutine
// stack.
type CallStack []Call
// String implements fmt.Stinger. It is equivalent to fmt.Sprintf("%v", cs).
func (cs CallStack) String() string {
return fmt.Sprint(cs)
}
var (
openBracketBytes = []byte("[")
closeBracketBytes = []byte("]")
spaceBytes = []byte(" ")
)
// MarshalText implements encoding.TextMarshaler. It formats the CallStack the
// same as fmt.Sprintf("%v", cs).
func (cs CallStack) MarshalText() ([]byte, error) {
buf := bytes.Buffer{}
buf.Write(openBracketBytes)
for i, pc := range cs {
if i > 0 {
buf.Write(spaceBytes)
}
fmt.Fprint(&buf, pc)
}
buf.Write(closeBracketBytes)
return buf.Bytes(), nil
}
// Format implements fmt.Formatter by printing the CallStack as square brackets
// ([, ]) surrounding a space separated list of Calls each formatted with the
// supplied verb and options.
func (cs CallStack) Format(s fmt.State, verb rune) {
s.Write(openBracketBytes)
for i, pc := range cs {
if i > 0 {
s.Write(spaceBytes)
}
pc.Format(s, verb)
}
s.Write(closeBracketBytes)
}
// Trace returns a CallStack for the current goroutine with element 0
// identifying the calling function.
func Trace() CallStack {
var pcs [512]uintptr
n := runtime.Callers(1, pcs[:])
frames := runtime.CallersFrames(pcs[:n])
cs := make(CallStack, 0, n)
// Skip extra frame retrieved just to make sure the runtime.sigpanic
// special case is handled.
frame, more := frames.Next()
for more {
frame, more = frames.Next()
cs = append(cs, Call{frame: frame})
}
return cs
}
// TrimBelow returns a slice of the CallStack with all entries below c
// removed.
func (cs CallStack) TrimBelow(c Call) CallStack {
for len(cs) > 0 && cs[0] != c {
cs = cs[1:]
}
return cs
}
// TrimAbove returns a slice of the CallStack with all entries above c
// removed.
func (cs CallStack) TrimAbove(c Call) CallStack {
for len(cs) > 0 && cs[len(cs)-1] != c {
cs = cs[:len(cs)-1]
}
return cs
}
// pkgIndex returns the index that results in file[index:] being the path of
// file relative to the compile time GOPATH, and file[:index] being the
// $GOPATH/src/ portion of file. funcName must be the name of a function in
// file as returned by runtime.Func.Name.
func pkgIndex(file, funcName string) int {
// As of Go 1.6.2 there is no direct way to know the compile time GOPATH
// at runtime, but we can infer the number of path segments in the GOPATH.
// We note that runtime.Func.Name() returns the function name qualified by
// the import path, which does not include the GOPATH. Thus we can trim
// segments from the beginning of the file path until the number of path
// separators remaining is one more than the number of path separators in
// the function name. For example, given:
//
// GOPATH /home/user
// file /home/user/src/pkg/sub/file.go
// fn.Name() pkg/sub.Type.Method
//
// We want to produce:
//
// file[:idx] == /home/user/src/
// file[idx:] == pkg/sub/file.go
//
// From this we can easily see that fn.Name() has one less path separator
// than our desired result for file[idx:]. We count separators from the
// end of the file path until it finds two more than in the function name
// and then move one character forward to preserve the initial path
// segment without a leading separator.
const sep = "/"
i := len(file)
for n := strings.Count(funcName, sep) + 2; n > 0; n-- {
i = strings.LastIndex(file[:i], sep)
if i == -1 {
i = -len(sep)
break
}
}
// get back to 0 or trim the leading separator
return i + len(sep)
}
// pkgFilePath returns the frame's filepath relative to the compile-time GOPATH,
// or its module path joined to its path relative to the module root.
//
// As of Go 1.11 there is no direct way to know the compile time GOPATH or
// module paths at runtime, but we can piece together the desired information
// from available information. We note that runtime.Frame.Function contains the
// function name qualified by the package path, which includes the module path
// but not the GOPATH. We can extract the package path from that and append the
// last segments of the file path to arrive at the desired package qualified
// file path. For example, given:
//
// GOPATH /home/user
// import path pkg/sub
// frame.File /home/user/src/pkg/sub/file.go
// frame.Function pkg/sub.Type.Method
// Desired return pkg/sub/file.go
//
// It appears that we simply need to trim ".Type.Method" from frame.Function and
// append "/" + path.Base(file).
//
// But there are other wrinkles. Although it is idiomatic to do so, the internal
// name of a package is not required to match the last segment of its import
// path. In addition, the introduction of modules in Go 1.11 allows working
// without a GOPATH. So we also must make these work right:
//
// GOPATH /home/user
// import path pkg/go-sub
// package name sub
// frame.File /home/user/src/pkg/go-sub/file.go
// frame.Function pkg/sub.Type.Method
// Desired return pkg/go-sub/file.go
//
// Module path pkg/v2
// import path pkg/v2/go-sub
// package name sub
// frame.File /home/user/cloned-pkg/go-sub/file.go
// frame.Function pkg/v2/sub.Type.Method
// Desired return pkg/v2/go-sub/file.go
//
// We can handle all of these situations by using the package path extracted
// from frame.Function up to, but not including, the last segment as the prefix
// and the last two segments of frame.File as the suffix of the returned path.
// This preserves the existing behavior when working in a GOPATH without modules
// and a semantically equivalent behavior when used in module aware project.
func pkgFilePath(frame *runtime.Frame) string {
pre := pkgPrefix(frame.Function)
post := pathSuffix(frame.File)
if pre == "" {
return post
}
return pre + "/" + post
}
// pkgPrefix returns the import path of the function's package with the final
// segment removed.
func pkgPrefix(funcName string) string {
const pathSep = "/"
end := strings.LastIndex(funcName, pathSep)
if end == -1 {
return ""
}
return funcName[:end]
}
// pathSuffix returns the last two segments of path.
func pathSuffix(path string) string {
const pathSep = "/"
lastSep := strings.LastIndex(path, pathSep)
if lastSep == -1 {
return path
}
return path[strings.LastIndex(path[:lastSep], pathSep)+1:]
}
var runtimePath string
func init() {
var pcs [3]uintptr
runtime.Callers(0, pcs[:])
frames := runtime.CallersFrames(pcs[:])
frame, _ := frames.Next()
file := frame.File
idx := pkgIndex(frame.File, frame.Function)
runtimePath = file[:idx]
if runtime.GOOS == "windows" {
runtimePath = strings.ToLower(runtimePath)
}
}
func inGoroot(c Call) bool {
file := c.frame.File
if len(file) == 0 || file[0] == '?' {
return true
}
if runtime.GOOS == "windows" {
file = strings.ToLower(file)
}
return strings.HasPrefix(file, runtimePath) || strings.HasSuffix(file, "/_testmain.go")
}
// TrimRuntime returns a slice of the CallStack with the topmost entries from
// the go runtime removed. It considers any calls originating from unknown
// files, files under GOROOT, or _testmain.go as part of the runtime.
func (cs CallStack) TrimRuntime() CallStack {
for len(cs) > 0 && inGoroot(cs[len(cs)-1]) {
cs = cs[:len(cs)-1]
}
return cs
}

16
vendor/github.com/golang/snappy/.gitignore generated vendored

@ -0,0 +1,16 @@
cmd/snappytool/snappytool
testdata/bench
# These explicitly listed benchmark data files are for an obsolete version of
# snappy_test.go.
testdata/alice29.txt
testdata/asyoulik.txt
testdata/fireworks.jpeg
testdata/geo.protodata
testdata/html
testdata/html_x_4
testdata/kppkn.gtb
testdata/lcet10.txt
testdata/paper-100k.pdf
testdata/plrabn12.txt
testdata/urls.10K

18
vendor/github.com/golang/snappy/AUTHORS generated vendored

@ -0,0 +1,18 @@
# This is the official list of Snappy-Go authors for copyright purposes.
# This file is distinct from the CONTRIBUTORS files.
# See the latter for an explanation.
# Names should be added to this file as
# Name or Organization <email address>
# The email address is not required for organizations.
# Please keep the list sorted.
Amazon.com, Inc
Damian Gryski <dgryski@gmail.com>
Eric Buth <eric@topos.com>
Google Inc.
Jan Mercl <0xjnml@gmail.com>
Klaus Post <klauspost@gmail.com>
Rodolfo Carvalho <rhcarvalho@gmail.com>
Sebastien Binet <seb.binet@gmail.com>

41
vendor/github.com/golang/snappy/CONTRIBUTORS generated vendored

@ -0,0 +1,41 @@
# This is the official list of people who can contribute
# (and typically have contributed) code to the Snappy-Go repository.
# The AUTHORS file lists the copyright holders; this file
# lists people. For example, Google employees are listed here
# but not in AUTHORS, because Google holds the copyright.
#
# The submission process automatically checks to make sure
# that people submitting code are listed in this file (by email address).
#
# Names should be added to this file only after verifying that
# the individual or the individual's organization has agreed to
# the appropriate Contributor License Agreement, found here:
#
# http://code.google.com/legal/individual-cla-v1.0.html
# http://code.google.com/legal/corporate-cla-v1.0.html
#
# The agreement for individuals can be filled out on the web.
#
# When adding J Random Contributor's name to this file,
# either J's name or J's organization's name should be
# added to the AUTHORS file, depending on whether the
# individual or corporate CLA was used.
# Names should be added to this file like so:
# Name <email address>
# Please keep the list sorted.
Alex Legg <alexlegg@google.com>
Damian Gryski <dgryski@gmail.com>
Eric Buth <eric@topos.com>
Jan Mercl <0xjnml@gmail.com>
Jonathan Swinney <jswinney@amazon.com>
Kai Backman <kaib@golang.org>
Klaus Post <klauspost@gmail.com>
Marc-Antoine Ruel <maruel@chromium.org>
Nigel Tao <nigeltao@golang.org>
Rob Pike <r@golang.org>
Rodolfo Carvalho <rhcarvalho@gmail.com>
Russ Cox <rsc@golang.org>
Sebastien Binet <seb.binet@gmail.com>

27
vendor/github.com/golang/snappy/LICENSE generated vendored

@ -0,0 +1,27 @@
Copyright (c) 2011 The Snappy-Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

107
vendor/github.com/golang/snappy/README generated vendored

@ -0,0 +1,107 @@
The Snappy compression format in the Go programming language.
To download and install from source:
$ go get github.com/golang/snappy
Unless otherwise noted, the Snappy-Go source files are distributed
under the BSD-style license found in the LICENSE file.
Benchmarks.
The golang/snappy benchmarks include compressing (Z) and decompressing (U) ten
or so files, the same set used by the C++ Snappy code (github.com/google/snappy
and note the "google", not "golang"). On an "Intel(R) Core(TM) i7-3770 CPU @
3.40GHz", Go's GOARCH=amd64 numbers as of 2016-05-29:
"go test -test.bench=."
_UFlat0-8 2.19GB/s ± 0% html
_UFlat1-8 1.41GB/s ± 0% urls
_UFlat2-8 23.5GB/s ± 2% jpg
_UFlat3-8 1.91GB/s ± 0% jpg_200
_UFlat4-8 14.0GB/s ± 1% pdf
_UFlat5-8 1.97GB/s ± 0% html4
_UFlat6-8 814MB/s ± 0% txt1
_UFlat7-8 785MB/s ± 0% txt2
_UFlat8-8 857MB/s ± 0% txt3
_UFlat9-8 719MB/s ± 1% txt4
_UFlat10-8 2.84GB/s ± 0% pb
_UFlat11-8 1.05GB/s ± 0% gaviota
_ZFlat0-8 1.04GB/s ± 0% html
_ZFlat1-8 534MB/s ± 0% urls
_ZFlat2-8 15.7GB/s ± 1% jpg
_ZFlat3-8 740MB/s ± 3% jpg_200
_ZFlat4-8 9.20GB/s ± 1% pdf
_ZFlat5-8 991MB/s ± 0% html4
_ZFlat6-8 379MB/s ± 0% txt1
_ZFlat7-8 352MB/s ± 0% txt2
_ZFlat8-8 396MB/s ± 1% txt3
_ZFlat9-8 327MB/s ± 1% txt4
_ZFlat10-8 1.33GB/s ± 1% pb
_ZFlat11-8 605MB/s ± 1% gaviota
"go test -test.bench=. -tags=noasm"
_UFlat0-8 621MB/s ± 2% html
_UFlat1-8 494MB/s ± 1% urls
_UFlat2-8 23.2GB/s ± 1% jpg
_UFlat3-8 1.12GB/s ± 1% jpg_200
_UFlat4-8 4.35GB/s ± 1% pdf
_UFlat5-8 609MB/s ± 0% html4
_UFlat6-8 296MB/s ± 0% txt1
_UFlat7-8 288MB/s ± 0% txt2
_UFlat8-8 309MB/s ± 1% txt3
_UFlat9-8 280MB/s ± 1% txt4
_UFlat10-8 753MB/s ± 0% pb
_UFlat11-8 400MB/s ± 0% gaviota
_ZFlat0-8 409MB/s ± 1% html
_ZFlat1-8 250MB/s ± 1% urls
_ZFlat2-8 12.3GB/s ± 1% jpg
_ZFlat3-8 132MB/s ± 0% jpg_200
_ZFlat4-8 2.92GB/s ± 0% pdf
_ZFlat5-8 405MB/s ± 1% html4
_ZFlat6-8 179MB/s ± 1% txt1
_ZFlat7-8 170MB/s ± 1% txt2
_ZFlat8-8 189MB/s ± 1% txt3
_ZFlat9-8 164MB/s ± 1% txt4
_ZFlat10-8 479MB/s ± 1% pb
_ZFlat11-8 270MB/s ± 1% gaviota
For comparison (Go's encoded output is byte-for-byte identical to C++'s), here
are the numbers from C++ Snappy's
make CXXFLAGS="-O2 -DNDEBUG -g" clean snappy_unittest.log && cat snappy_unittest.log
BM_UFlat/0 2.4GB/s html
BM_UFlat/1 1.4GB/s urls
BM_UFlat/2 21.8GB/s jpg
BM_UFlat/3 1.5GB/s jpg_200
BM_UFlat/4 13.3GB/s pdf
BM_UFlat/5 2.1GB/s html4
BM_UFlat/6 1.0GB/s txt1
BM_UFlat/7 959.4MB/s txt2
BM_UFlat/8 1.0GB/s txt3
BM_UFlat/9 864.5MB/s txt4
BM_UFlat/10 2.9GB/s pb
BM_UFlat/11 1.2GB/s gaviota
BM_ZFlat/0 944.3MB/s html (22.31 %)
BM_ZFlat/1 501.6MB/s urls (47.78 %)
BM_ZFlat/2 14.3GB/s jpg (99.95 %)
BM_ZFlat/3 538.3MB/s jpg_200 (73.00 %)
BM_ZFlat/4 8.3GB/s pdf (83.30 %)
BM_ZFlat/5 903.5MB/s html4 (22.52 %)
BM_ZFlat/6 336.0MB/s txt1 (57.88 %)
BM_ZFlat/7 312.3MB/s txt2 (61.91 %)
BM_ZFlat/8 353.1MB/s txt3 (54.99 %)
BM_ZFlat/9 289.9MB/s txt4 (66.26 %)
BM_ZFlat/10 1.2GB/s pb (19.68 %)
BM_ZFlat/11 527.4MB/s gaviota (37.72 %)

264
vendor/github.com/golang/snappy/decode.go generated vendored

@ -0,0 +1,264 @@
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package snappy
import (
"encoding/binary"
"errors"
"io"
)
var (
// ErrCorrupt reports that the input is invalid.
ErrCorrupt = errors.New("snappy: corrupt input")
// ErrTooLarge reports that the uncompressed length is too large.
ErrTooLarge = errors.New("snappy: decoded block is too large")
// ErrUnsupported reports that the input isn't supported.
ErrUnsupported = errors.New("snappy: unsupported input")
errUnsupportedLiteralLength = errors.New("snappy: unsupported literal length")
)
// DecodedLen returns the length of the decoded block.
func DecodedLen(src []byte) (int, error) {
v, _, err := decodedLen(src)
return v, err
}
// decodedLen returns the length of the decoded block and the number of bytes
// that the length header occupied.
func decodedLen(src []byte) (blockLen, headerLen int, err error) {
v, n := binary.Uvarint(src)
if n <= 0 || v > 0xffffffff {
return 0, 0, ErrCorrupt
}
const wordSize = 32 << (^uint(0) >> 32 & 1)
if wordSize == 32 && v > 0x7fffffff {
return 0, 0, ErrTooLarge
}
return int(v), n, nil
}
const (
decodeErrCodeCorrupt = 1
decodeErrCodeUnsupportedLiteralLength = 2
)
// Decode returns the decoded form of src. The returned slice may be a sub-
// slice of dst if dst was large enough to hold the entire decoded block.
// Otherwise, a newly allocated slice will be returned.
//
// The dst and src must not overlap. It is valid to pass a nil dst.
//
// Decode handles the Snappy block format, not the Snappy stream format.
func Decode(dst, src []byte) ([]byte, error) {
dLen, s, err := decodedLen(src)
if err != nil {
return nil, err
}
if dLen <= len(dst) {
dst = dst[:dLen]
} else {
dst = make([]byte, dLen)
}
switch decode(dst, src[s:]) {
case 0:
return dst, nil
case decodeErrCodeUnsupportedLiteralLength:
return nil, errUnsupportedLiteralLength
}
return nil, ErrCorrupt
}
// NewReader returns a new Reader that decompresses from r, using the framing
// format described at
// https://github.com/google/snappy/blob/master/framing_format.txt
func NewReader(r io.Reader) *Reader {
return &Reader{
r: r,
decoded: make([]byte, maxBlockSize),
buf: make([]byte, maxEncodedLenOfMaxBlockSize+checksumSize),
}
}
// Reader is an io.Reader that can read Snappy-compressed bytes.
//
// Reader handles the Snappy stream format, not the Snappy block format.
type Reader struct {
r io.Reader
err error
decoded []byte
buf []byte
// decoded[i:j] contains decoded bytes that have not yet been passed on.
i, j int
readHeader bool
}
// Reset discards any buffered data, resets all state, and switches the Snappy
// reader to read from r. This permits reusing a Reader rather than allocating
// a new one.
func (r *Reader) Reset(reader io.Reader) {
r.r = reader
r.err = nil
r.i = 0
r.j = 0
r.readHeader = false
}
func (r *Reader) readFull(p []byte, allowEOF bool) (ok bool) {
if _, r.err = io.ReadFull(r.r, p); r.err != nil {
if r.err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) {
r.err = ErrCorrupt
}
return false
}
return true
}
func (r *Reader) fill() error {
for r.i >= r.j {
if !r.readFull(r.buf[:4], true) {
return r.err
}
chunkType := r.buf[0]
if !r.readHeader {
if chunkType != chunkTypeStreamIdentifier {
r.err = ErrCorrupt
return r.err
}
r.readHeader = true
}
chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16
if chunkLen > len(r.buf) {
r.err = ErrUnsupported
return r.err
}
// The chunk types are specified at
// https://github.com/google/snappy/blob/master/framing_format.txt
switch chunkType {
case chunkTypeCompressedData:
// Section 4.2. Compressed data (chunk type 0x00).
if chunkLen < checksumSize {
r.err = ErrCorrupt
return r.err
}
buf := r.buf[:chunkLen]
if !r.readFull(buf, false) {
return r.err
}
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
buf = buf[checksumSize:]
n, err := DecodedLen(buf)
if err != nil {
r.err = err
return r.err
}
if n > len(r.decoded) {
r.err = ErrCorrupt
return r.err
}
if _, err := Decode(r.decoded, buf); err != nil {
r.err = err
return r.err
}
if crc(r.decoded[:n]) != checksum {
r.err = ErrCorrupt
return r.err
}
r.i, r.j = 0, n
continue
case chunkTypeUncompressedData:
// Section 4.3. Uncompressed data (chunk type 0x01).
if chunkLen < checksumSize {
r.err = ErrCorrupt
return r.err
}
buf := r.buf[:checksumSize]
if !r.readFull(buf, false) {
return r.err
}
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
// Read directly into r.decoded instead of via r.buf.
n := chunkLen - checksumSize
if n > len(r.decoded) {
r.err = ErrCorrupt
return r.err
}
if !r.readFull(r.decoded[:n], false) {
return r.err
}
if crc(r.decoded[:n]) != checksum {
r.err = ErrCorrupt
return r.err
}
r.i, r.j = 0, n
continue
case chunkTypeStreamIdentifier:
// Section 4.1. Stream identifier (chunk type 0xff).
if chunkLen != len(magicBody) {
r.err = ErrCorrupt
return r.err
}
if !r.readFull(r.buf[:len(magicBody)], false) {
return r.err
}
for i := 0; i < len(magicBody); i++ {
if r.buf[i] != magicBody[i] {
r.err = ErrCorrupt
return r.err
}
}
continue
}
if chunkType <= 0x7f {
// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
r.err = ErrUnsupported
return r.err
}
// Section 4.4 Padding (chunk type 0xfe).
// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
if !r.readFull(r.buf[:chunkLen], false) {
return r.err
}
}
return nil
}
// Read satisfies the io.Reader interface.
func (r *Reader) Read(p []byte) (int, error) {
if r.err != nil {
return 0, r.err
}
if err := r.fill(); err != nil {
return 0, err
}
n := copy(p, r.decoded[r.i:r.j])
r.i += n
return n, nil
}
// ReadByte satisfies the io.ByteReader interface.
func (r *Reader) ReadByte() (byte, error) {
if r.err != nil {
return 0, r.err
}
if err := r.fill(); err != nil {
return 0, err
}
c := r.decoded[r.i]
r.i++
return c, nil
}

490
vendor/github.com/golang/snappy/decode_amd64.s generated vendored

@ -0,0 +1,490 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !appengine
// +build gc
// +build !noasm
#include "textflag.h"
// The asm code generally follows the pure Go code in decode_other.go, except
// where marked with a "!!!".
// func decode(dst, src []byte) int
//
// All local variables fit into registers. The non-zero stack size is only to
// spill registers and push args when issuing a CALL. The register allocation:
// - AX scratch
// - BX scratch
// - CX length or x
// - DX offset
// - SI &src[s]
// - DI &dst[d]
// + R8 dst_base
// + R9 dst_len
// + R10 dst_base + dst_len
// + R11 src_base
// + R12 src_len
// + R13 src_base + src_len
// - R14 used by doCopy
// - R15 used by doCopy
//
// The registers R8-R13 (marked with a "+") are set at the start of the
// function, and after a CALL returns, and are not otherwise modified.
//
// The d variable is implicitly DI - R8, and len(dst)-d is R10 - DI.
// The s variable is implicitly SI - R11, and len(src)-s is R13 - SI.
TEXT ·decode(SB), NOSPLIT, $48-56
// Initialize SI, DI and R8-R13.
MOVQ dst_base+0(FP), R8
MOVQ dst_len+8(FP), R9
MOVQ R8, DI
MOVQ R8, R10
ADDQ R9, R10
MOVQ src_base+24(FP), R11
MOVQ src_len+32(FP), R12
MOVQ R11, SI
MOVQ R11, R13
ADDQ R12, R13
loop:
// for s < len(src)
CMPQ SI, R13
JEQ end
// CX = uint32(src[s])
//
// switch src[s] & 0x03
MOVBLZX (SI), CX
MOVL CX, BX
ANDL $3, BX
CMPL BX, $1
JAE tagCopy
// ----------------------------------------
// The code below handles literal tags.
// case tagLiteral:
// x := uint32(src[s] >> 2)
// switch
SHRL $2, CX
CMPL CX, $60
JAE tagLit60Plus
// case x < 60:
// s++
INCQ SI
doLit:
// This is the end of the inner "switch", when we have a literal tag.
//
// We assume that CX == x and x fits in a uint32, where x is the variable
// used in the pure Go decode_other.go code.
// length = int(x) + 1
//
// Unlike the pure Go code, we don't need to check if length <= 0 because
// CX can hold 64 bits, so the increment cannot overflow.
INCQ CX
// Prepare to check if copying length bytes will run past the end of dst or
// src.
//
// AX = len(dst) - d
// BX = len(src) - s
MOVQ R10, AX
SUBQ DI, AX
MOVQ R13, BX
SUBQ SI, BX
// !!! Try a faster technique for short (16 or fewer bytes) copies.
//
// if length > 16 || len(dst)-d < 16 || len(src)-s < 16 {
// goto callMemmove // Fall back on calling runtime·memmove.
// }
//
// The C++ snappy code calls this TryFastAppend. It also checks len(src)-s
// against 21 instead of 16, because it cannot assume that all of its input
// is contiguous in memory and so it needs to leave enough source bytes to
// read the next tag without refilling buffers, but Go's Decode assumes
// contiguousness (the src argument is a []byte).
CMPQ CX, $16
JGT callMemmove
CMPQ AX, $16
JLT callMemmove
CMPQ BX, $16
JLT callMemmove
// !!! Implement the copy from src to dst as a 16-byte load and store.
// (Decode's documentation says that dst and src must not overlap.)
//
// This always copies 16 bytes, instead of only length bytes, but that's
// OK. If the input is a valid Snappy encoding then subsequent iterations
// will fix up the overrun. Otherwise, Decode returns a nil []byte (and a
// non-nil error), so the overrun will be ignored.
//
// Note that on amd64, it is legal and cheap to issue unaligned 8-byte or
// 16-byte loads and stores. This technique probably wouldn't be as
// effective on architectures that are fussier about alignment.
MOVOU 0(SI), X0
MOVOU X0, 0(DI)
// d += length
// s += length
ADDQ CX, DI
ADDQ CX, SI
JMP loop
callMemmove:
// if length > len(dst)-d || length > len(src)-s { etc }
CMPQ CX, AX
JGT errCorrupt
CMPQ CX, BX
JGT errCorrupt
// copy(dst[d:], src[s:s+length])
//
// This means calling runtime·memmove(&dst[d], &src[s], length), so we push
// DI, SI and CX as arguments. Coincidentally, we also need to spill those
// three registers to the stack, to save local variables across the CALL.
MOVQ DI, 0(SP)
MOVQ SI, 8(SP)
MOVQ CX, 16(SP)
MOVQ DI, 24(SP)
MOVQ SI, 32(SP)
MOVQ CX, 40(SP)
CALL runtime·memmove(SB)
// Restore local variables: unspill registers from the stack and
// re-calculate R8-R13.
MOVQ 24(SP), DI
MOVQ 32(SP), SI
MOVQ 40(SP), CX
MOVQ dst_base+0(FP), R8
MOVQ dst_len+8(FP), R9
MOVQ R8, R10
ADDQ R9, R10
MOVQ src_base+24(FP), R11
MOVQ src_len+32(FP), R12
MOVQ R11, R13
ADDQ R12, R13
// d += length
// s += length
ADDQ CX, DI
ADDQ CX, SI
JMP loop
tagLit60Plus:
// !!! This fragment does the
//
// s += x - 58; if uint(s) > uint(len(src)) { etc }
//
// checks. In the asm version, we code it once instead of once per switch case.
ADDQ CX, SI
SUBQ $58, SI
MOVQ SI, BX
SUBQ R11, BX
CMPQ BX, R12
JA errCorrupt
// case x == 60:
CMPL CX, $61
JEQ tagLit61
JA tagLit62Plus
// x = uint32(src[s-1])
MOVBLZX -1(SI), CX
JMP doLit
tagLit61:
// case x == 61:
// x = uint32(src[s-2]) | uint32(src[s-1])<<8
MOVWLZX -2(SI), CX
JMP doLit
tagLit62Plus:
CMPL CX, $62
JA tagLit63
// case x == 62:
// x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
MOVWLZX -3(SI), CX
MOVBLZX -1(SI), BX
SHLL $16, BX
ORL BX, CX
JMP doLit
tagLit63:
// case x == 63:
// x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
MOVL -4(SI), CX
JMP doLit
// The code above handles literal tags.
// ----------------------------------------
// The code below handles copy tags.
tagCopy4:
// case tagCopy4:
// s += 5
ADDQ $5, SI
// if uint(s) > uint(len(src)) { etc }
MOVQ SI, BX
SUBQ R11, BX
CMPQ BX, R12
JA errCorrupt
// length = 1 + int(src[s-5])>>2
SHRQ $2, CX
INCQ CX
// offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
MOVLQZX -4(SI), DX
JMP doCopy
tagCopy2:
// case tagCopy2:
// s += 3
ADDQ $3, SI
// if uint(s) > uint(len(src)) { etc }
MOVQ SI, BX
SUBQ R11, BX
CMPQ BX, R12
JA errCorrupt
// length = 1 + int(src[s-3])>>2
SHRQ $2, CX
INCQ CX
// offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
MOVWQZX -2(SI), DX
JMP doCopy
tagCopy:
// We have a copy tag. We assume that:
// - BX == src[s] & 0x03
// - CX == src[s]
CMPQ BX, $2
JEQ tagCopy2
JA tagCopy4
// case tagCopy1:
// s += 2
ADDQ $2, SI
// if uint(s) > uint(len(src)) { etc }
MOVQ SI, BX
SUBQ R11, BX
CMPQ BX, R12
JA errCorrupt
// offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
MOVQ CX, DX
ANDQ $0xe0, DX
SHLQ $3, DX
MOVBQZX -1(SI), BX
ORQ BX, DX
// length = 4 + int(src[s-2])>>2&0x7
SHRQ $2, CX
ANDQ $7, CX
ADDQ $4, CX
doCopy:
// This is the end of the outer "switch", when we have a copy tag.
//
// We assume that:
// - CX == length && CX > 0
// - DX == offset
// if offset <= 0 { etc }
CMPQ DX, $0
JLE errCorrupt
// if d < offset { etc }
MOVQ DI, BX
SUBQ R8, BX
CMPQ BX, DX
JLT errCorrupt
// if length > len(dst)-d { etc }
MOVQ R10, BX
SUBQ DI, BX
CMPQ CX, BX
JGT errCorrupt
// forwardCopy(dst[d:d+length], dst[d-offset:]); d += length
//
// Set:
// - R14 = len(dst)-d
// - R15 = &dst[d-offset]
MOVQ R10, R14
SUBQ DI, R14
MOVQ DI, R15
SUBQ DX, R15
// !!! Try a faster technique for short (16 or fewer bytes) forward copies.
//
// First, try using two 8-byte load/stores, similar to the doLit technique
// above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is
// still OK if offset >= 8. Note that this has to be two 8-byte load/stores
// and not one 16-byte load/store, and the first store has to be before the
// second load, due to the overlap if offset is in the range [8, 16).
//
// if length > 16 || offset < 8 || len(dst)-d < 16 {
// goto slowForwardCopy
// }
// copy 16 bytes
// d += length
CMPQ CX, $16
JGT slowForwardCopy
CMPQ DX, $8
JLT slowForwardCopy
CMPQ R14, $16
JLT slowForwardCopy
MOVQ 0(R15), AX
MOVQ AX, 0(DI)
MOVQ 8(R15), BX
MOVQ BX, 8(DI)
ADDQ CX, DI
JMP loop
slowForwardCopy:
// !!! If the forward copy is longer than 16 bytes, or if offset < 8, we
// can still try 8-byte load stores, provided we can overrun up to 10 extra
// bytes. As above, the overrun will be fixed up by subsequent iterations
// of the outermost loop.
//
// The C++ snappy code calls this technique IncrementalCopyFastPath. Its
// commentary says:
//
// ----
//
// The main part of this loop is a simple copy of eight bytes at a time
// until we've copied (at least) the requested amount of bytes. However,
// if d and d-offset are less than eight bytes apart (indicating a
// repeating pattern of length < 8), we first need to expand the pattern in
// order to get the correct results. For instance, if the buffer looks like
// this, with the eight-byte <d-offset> and <d> patterns marked as
// intervals:
//
// abxxxxxxxxxxxx
// [------] d-offset
// [------] d
//
// a single eight-byte copy from <d-offset> to <d> will repeat the pattern
// once, after which we can move <d> two bytes without moving <d-offset>:
//
// ababxxxxxxxxxx
// [------] d-offset
// [------] d
//
// and repeat the exercise until the two no longer overlap.
//
// This allows us to do very well in the special case of one single byte
// repeated many times, without taking a big hit for more general cases.
//
// The worst case of extra writing past the end of the match occurs when
// offset == 1 and length == 1; the last copy will read from byte positions
// [0..7] and write to [4..11], whereas it was only supposed to write to
// position 1. Thus, ten excess bytes.
//
// ----
//
// That "10 byte overrun" worst case is confirmed by Go's
// TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy
// and finishSlowForwardCopy algorithm.
//
// if length > len(dst)-d-10 {
// goto verySlowForwardCopy
// }
SUBQ $10, R14
CMPQ CX, R14
JGT verySlowForwardCopy
makeOffsetAtLeast8:
// !!! As above, expand the pattern so that offset >= 8 and we can use
// 8-byte load/stores.
//
// for offset < 8 {
// copy 8 bytes from dst[d-offset:] to dst[d:]
// length -= offset
// d += offset
// offset += offset
// // The two previous lines together means that d-offset, and therefore
// // R15, is unchanged.
// }
CMPQ DX, $8
JGE fixUpSlowForwardCopy
MOVQ (R15), BX
MOVQ BX, (DI)
SUBQ DX, CX
ADDQ DX, DI
ADDQ DX, DX
JMP makeOffsetAtLeast8
fixUpSlowForwardCopy:
// !!! Add length (which might be negative now) to d (implied by DI being
// &dst[d]) so that d ends up at the right place when we jump back to the
// top of the loop. Before we do that, though, we save DI to AX so that, if
// length is positive, copying the remaining length bytes will write to the
// right place.
MOVQ DI, AX
ADDQ CX, DI
finishSlowForwardCopy:
// !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative
// length means that we overrun, but as above, that will be fixed up by
// subsequent iterations of the outermost loop.
CMPQ CX, $0
JLE loop
MOVQ (R15), BX
MOVQ BX, (AX)
ADDQ $8, R15
ADDQ $8, AX
SUBQ $8, CX
JMP finishSlowForwardCopy
verySlowForwardCopy:
// verySlowForwardCopy is a simple implementation of forward copy. In C
// parlance, this is a do/while loop instead of a while loop, since we know
// that length > 0. In Go syntax:
//
// for {
// dst[d] = dst[d - offset]
// d++
// length--
// if length == 0 {
// break
// }
// }
MOVB (R15), BX
MOVB BX, (DI)
INCQ R15
INCQ DI
DECQ CX
JNZ verySlowForwardCopy
JMP loop
// The code above handles copy tags.
// ----------------------------------------
end:
// This is the end of the "for s < len(src)".
//
// if d != len(dst) { etc }
CMPQ DI, R10
JNE errCorrupt
// return 0
MOVQ $0, ret+48(FP)
RET
errCorrupt:
// return decodeErrCodeCorrupt
MOVQ $1, ret+48(FP)
RET

494
vendor/github.com/golang/snappy/decode_arm64.s generated vendored

@ -0,0 +1,494 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !appengine
// +build gc
// +build !noasm
#include "textflag.h"
// The asm code generally follows the pure Go code in decode_other.go, except
// where marked with a "!!!".
// func decode(dst, src []byte) int
//
// All local variables fit into registers. The non-zero stack size is only to
// spill registers and push args when issuing a CALL. The register allocation:
// - R2 scratch
// - R3 scratch
// - R4 length or x
// - R5 offset
// - R6 &src[s]
// - R7 &dst[d]
// + R8 dst_base
// + R9 dst_len
// + R10 dst_base + dst_len
// + R11 src_base
// + R12 src_len
// + R13 src_base + src_len
// - R14 used by doCopy
// - R15 used by doCopy
//
// The registers R8-R13 (marked with a "+") are set at the start of the
// function, and after a CALL returns, and are not otherwise modified.
//
// The d variable is implicitly R7 - R8, and len(dst)-d is R10 - R7.
// The s variable is implicitly R6 - R11, and len(src)-s is R13 - R6.
TEXT ·decode(SB), NOSPLIT, $56-56
// Initialize R6, R7 and R8-R13.
MOVD dst_base+0(FP), R8
MOVD dst_len+8(FP), R9
MOVD R8, R7
MOVD R8, R10
ADD R9, R10, R10
MOVD src_base+24(FP), R11
MOVD src_len+32(FP), R12
MOVD R11, R6
MOVD R11, R13
ADD R12, R13, R13
loop:
// for s < len(src)
CMP R13, R6
BEQ end
// R4 = uint32(src[s])
//
// switch src[s] & 0x03
MOVBU (R6), R4
MOVW R4, R3
ANDW $3, R3
MOVW $1, R1
CMPW R1, R3
BGE tagCopy
// ----------------------------------------
// The code below handles literal tags.
// case tagLiteral:
// x := uint32(src[s] >> 2)
// switch
MOVW $60, R1
LSRW $2, R4, R4
CMPW R4, R1
BLS tagLit60Plus
// case x < 60:
// s++
ADD $1, R6, R6
doLit:
// This is the end of the inner "switch", when we have a literal tag.
//
// We assume that R4 == x and x fits in a uint32, where x is the variable
// used in the pure Go decode_other.go code.
// length = int(x) + 1
//
// Unlike the pure Go code, we don't need to check if length <= 0 because
// R4 can hold 64 bits, so the increment cannot overflow.
ADD $1, R4, R4
// Prepare to check if copying length bytes will run past the end of dst or
// src.
//
// R2 = len(dst) - d
// R3 = len(src) - s
MOVD R10, R2
SUB R7, R2, R2
MOVD R13, R3
SUB R6, R3, R3
// !!! Try a faster technique for short (16 or fewer bytes) copies.
//
// if length > 16 || len(dst)-d < 16 || len(src)-s < 16 {
// goto callMemmove // Fall back on calling runtime·memmove.
// }
//
// The C++ snappy code calls this TryFastAppend. It also checks len(src)-s
// against 21 instead of 16, because it cannot assume that all of its input
// is contiguous in memory and so it needs to leave enough source bytes to
// read the next tag without refilling buffers, but Go's Decode assumes
// contiguousness (the src argument is a []byte).
CMP $16, R4
BGT callMemmove
CMP $16, R2
BLT callMemmove
CMP $16, R3
BLT callMemmove
// !!! Implement the copy from src to dst as a 16-byte load and store.
// (Decode's documentation says that dst and src must not overlap.)
//
// This always copies 16 bytes, instead of only length bytes, but that's
// OK. If the input is a valid Snappy encoding then subsequent iterations
// will fix up the overrun. Otherwise, Decode returns a nil []byte (and a
// non-nil error), so the overrun will be ignored.
//
// Note that on arm64, it is legal and cheap to issue unaligned 8-byte or
// 16-byte loads and stores. This technique probably wouldn't be as
// effective on architectures that are fussier about alignment.
LDP 0(R6), (R14, R15)
STP (R14, R15), 0(R7)
// d += length
// s += length
ADD R4, R7, R7
ADD R4, R6, R6
B loop
callMemmove:
// if length > len(dst)-d || length > len(src)-s { etc }
CMP R2, R4
BGT errCorrupt
CMP R3, R4
BGT errCorrupt
// copy(dst[d:], src[s:s+length])
//
// This means calling runtime·memmove(&dst[d], &src[s], length), so we push
// R7, R6 and R4 as arguments. Coincidentally, we also need to spill those
// three registers to the stack, to save local variables across the CALL.
MOVD R7, 8(RSP)
MOVD R6, 16(RSP)
MOVD R4, 24(RSP)
MOVD R7, 32(RSP)
MOVD R6, 40(RSP)
MOVD R4, 48(RSP)
CALL runtime·memmove(SB)
// Restore local variables: unspill registers from the stack and
// re-calculate R8-R13.
MOVD 32(RSP), R7
MOVD 40(RSP), R6
MOVD 48(RSP), R4
MOVD dst_base+0(FP), R8
MOVD dst_len+8(FP), R9
MOVD R8, R10
ADD R9, R10, R10
MOVD src_base+24(FP), R11
MOVD src_len+32(FP), R12
MOVD R11, R13
ADD R12, R13, R13
// d += length
// s += length
ADD R4, R7, R7
ADD R4, R6, R6
B loop
tagLit60Plus:
// !!! This fragment does the
//
// s += x - 58; if uint(s) > uint(len(src)) { etc }
//
// checks. In the asm version, we code it once instead of once per switch case.
ADD R4, R6, R6
SUB $58, R6, R6
MOVD R6, R3
SUB R11, R3, R3
CMP R12, R3
BGT errCorrupt
// case x == 60:
MOVW $61, R1
CMPW R1, R4
BEQ tagLit61
BGT tagLit62Plus
// x = uint32(src[s-1])
MOVBU -1(R6), R4
B doLit
tagLit61:
// case x == 61:
// x = uint32(src[s-2]) | uint32(src[s-1])<<8
MOVHU -2(R6), R4
B doLit
tagLit62Plus:
CMPW $62, R4
BHI tagLit63
// case x == 62:
// x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
MOVHU -3(R6), R4
MOVBU -1(R6), R3
ORR R3<<16, R4
B doLit
tagLit63:
// case x == 63:
// x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
MOVWU -4(R6), R4
B doLit
// The code above handles literal tags.
// ----------------------------------------
// The code below handles copy tags.
tagCopy4:
// case tagCopy4:
// s += 5
ADD $5, R6, R6
// if uint(s) > uint(len(src)) { etc }
MOVD R6, R3
SUB R11, R3, R3
CMP R12, R3
BGT errCorrupt
// length = 1 + int(src[s-5])>>2
MOVD $1, R1
ADD R4>>2, R1, R4
// offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
MOVWU -4(R6), R5
B doCopy
tagCopy2:
// case tagCopy2:
// s += 3
ADD $3, R6, R6
// if uint(s) > uint(len(src)) { etc }
MOVD R6, R3
SUB R11, R3, R3
CMP R12, R3
BGT errCorrupt
// length = 1 + int(src[s-3])>>2
MOVD $1, R1
ADD R4>>2, R1, R4
// offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
MOVHU -2(R6), R5
B doCopy
tagCopy:
// We have a copy tag. We assume that:
// - R3 == src[s] & 0x03
// - R4 == src[s]
CMP $2, R3
BEQ tagCopy2
BGT tagCopy4
// case tagCopy1:
// s += 2
ADD $2, R6, R6
// if uint(s) > uint(len(src)) { etc }
MOVD R6, R3
SUB R11, R3, R3
CMP R12, R3
BGT errCorrupt
// offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
MOVD R4, R5
AND $0xe0, R5
MOVBU -1(R6), R3
ORR R5<<3, R3, R5
// length = 4 + int(src[s-2])>>2&0x7
MOVD $7, R1
AND R4>>2, R1, R4
ADD $4, R4, R4
doCopy:
// This is the end of the outer "switch", when we have a copy tag.
//
// We assume that:
// - R4 == length && R4 > 0
// - R5 == offset
// if offset <= 0 { etc }
MOVD $0, R1
CMP R1, R5
BLE errCorrupt
// if d < offset { etc }
MOVD R7, R3
SUB R8, R3, R3
CMP R5, R3
BLT errCorrupt
// if length > len(dst)-d { etc }
MOVD R10, R3
SUB R7, R3, R3
CMP R3, R4
BGT errCorrupt
// forwardCopy(dst[d:d+length], dst[d-offset:]); d += length
//
// Set:
// - R14 = len(dst)-d
// - R15 = &dst[d-offset]
MOVD R10, R14
SUB R7, R14, R14
MOVD R7, R15
SUB R5, R15, R15
// !!! Try a faster technique for short (16 or fewer bytes) forward copies.
//
// First, try using two 8-byte load/stores, similar to the doLit technique
// above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is
// still OK if offset >= 8. Note that this has to be two 8-byte load/stores
// and not one 16-byte load/store, and the first store has to be before the
// second load, due to the overlap if offset is in the range [8, 16).
//
// if length > 16 || offset < 8 || len(dst)-d < 16 {
// goto slowForwardCopy
// }
// copy 16 bytes
// d += length
CMP $16, R4
BGT slowForwardCopy
CMP $8, R5
BLT slowForwardCopy
CMP $16, R14
BLT slowForwardCopy
MOVD 0(R15), R2
MOVD R2, 0(R7)
MOVD 8(R15), R3
MOVD R3, 8(R7)
ADD R4, R7, R7
B loop
slowForwardCopy:
// !!! If the forward copy is longer than 16 bytes, or if offset < 8, we
// can still try 8-byte load stores, provided we can overrun up to 10 extra
// bytes. As above, the overrun will be fixed up by subsequent iterations
// of the outermost loop.
//
// The C++ snappy code calls this technique IncrementalCopyFastPath. Its
// commentary says:
//
// ----
//
// The main part of this loop is a simple copy of eight bytes at a time
// until we've copied (at least) the requested amount of bytes. However,
// if d and d-offset are less than eight bytes apart (indicating a
// repeating pattern of length < 8), we first need to expand the pattern in
// order to get the correct results. For instance, if the buffer looks like
// this, with the eight-byte <d-offset> and <d> patterns marked as
// intervals:
//
// abxxxxxxxxxxxx
// [------] d-offset
// [------] d
//
// a single eight-byte copy from <d-offset> to <d> will repeat the pattern
// once, after which we can move <d> two bytes without moving <d-offset>:
//
// ababxxxxxxxxxx
// [------] d-offset
// [------] d
//
// and repeat the exercise until the two no longer overlap.
//
// This allows us to do very well in the special case of one single byte
// repeated many times, without taking a big hit for more general cases.
//
// The worst case of extra writing past the end of the match occurs when
// offset == 1 and length == 1; the last copy will read from byte positions
// [0..7] and write to [4..11], whereas it was only supposed to write to
// position 1. Thus, ten excess bytes.
//
// ----
//
// That "10 byte overrun" worst case is confirmed by Go's
// TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy
// and finishSlowForwardCopy algorithm.
//
// if length > len(dst)-d-10 {
// goto verySlowForwardCopy
// }
SUB $10, R14, R14
CMP R14, R4
BGT verySlowForwardCopy
makeOffsetAtLeast8:
// !!! As above, expand the pattern so that offset >= 8 and we can use
// 8-byte load/stores.
//
// for offset < 8 {
// copy 8 bytes from dst[d-offset:] to dst[d:]
// length -= offset
// d += offset
// offset += offset
// // The two previous lines together means that d-offset, and therefore
// // R15, is unchanged.
// }
CMP $8, R5
BGE fixUpSlowForwardCopy
MOVD (R15), R3
MOVD R3, (R7)
SUB R5, R4, R4
ADD R5, R7, R7
ADD R5, R5, R5
B makeOffsetAtLeast8
fixUpSlowForwardCopy:
// !!! Add length (which might be negative now) to d (implied by R7 being
// &dst[d]) so that d ends up at the right place when we jump back to the
// top of the loop. Before we do that, though, we save R7 to R2 so that, if
// length is positive, copying the remaining length bytes will write to the
// right place.
MOVD R7, R2
ADD R4, R7, R7
finishSlowForwardCopy:
// !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative
// length means that we overrun, but as above, that will be fixed up by
// subsequent iterations of the outermost loop.
MOVD $0, R1
CMP R1, R4
BLE loop
MOVD (R15), R3
MOVD R3, (R2)
ADD $8, R15, R15
ADD $8, R2, R2
SUB $8, R4, R4
B finishSlowForwardCopy
verySlowForwardCopy:
// verySlowForwardCopy is a simple implementation of forward copy. In C
// parlance, this is a do/while loop instead of a while loop, since we know
// that length > 0. In Go syntax:
//
// for {
// dst[d] = dst[d - offset]
// d++
// length--
// if length == 0 {
// break
// }
// }
MOVB (R15), R3
MOVB R3, (R7)
ADD $1, R15, R15
ADD $1, R7, R7
SUB $1, R4, R4
CBNZ R4, verySlowForwardCopy
B loop
// The code above handles copy tags.
// ----------------------------------------
end:
// This is the end of the "for s < len(src)".
//
// if d != len(dst) { etc }
CMP R10, R7
BNE errCorrupt
// return 0
MOVD $0, ret+48(FP)
RET
errCorrupt:
// return decodeErrCodeCorrupt
MOVD $1, R2
MOVD R2, ret+48(FP)
RET

15
vendor/github.com/golang/snappy/decode_asm.go generated vendored

@ -0,0 +1,15 @@
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !appengine
// +build gc
// +build !noasm
// +build amd64 arm64
package snappy
// decode has the same semantics as in decode_other.go.
//
//go:noescape
func decode(dst, src []byte) int

115
vendor/github.com/golang/snappy/decode_other.go generated vendored

@ -0,0 +1,115 @@
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !amd64,!arm64 appengine !gc noasm
package snappy
// decode writes the decoding of src to dst. It assumes that the varint-encoded
// length of the decompressed bytes has already been read, and that len(dst)
// equals that length.
//
// It returns 0 on success or a decodeErrCodeXxx error code on failure.
func decode(dst, src []byte) int {
var d, s, offset, length int
for s < len(src) {
switch src[s] & 0x03 {
case tagLiteral:
x := uint32(src[s] >> 2)
switch {
case x < 60:
s++
case x == 60:
s += 2
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
return decodeErrCodeCorrupt
}
x = uint32(src[s-1])
case x == 61:
s += 3
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
return decodeErrCodeCorrupt
}
x = uint32(src[s-2]) | uint32(src[s-1])<<8
case x == 62:
s += 4
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
return decodeErrCodeCorrupt
}
x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
case x == 63:
s += 5
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
return decodeErrCodeCorrupt
}
x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
}
length = int(x) + 1
if length <= 0 {
return decodeErrCodeUnsupportedLiteralLength
}
if length > len(dst)-d || length > len(src)-s {
return decodeErrCodeCorrupt
}
copy(dst[d:], src[s:s+length])
d += length
s += length
continue
case tagCopy1:
s += 2
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
return decodeErrCodeCorrupt
}
length = 4 + int(src[s-2])>>2&0x7
offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
case tagCopy2:
s += 3
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
return decodeErrCodeCorrupt
}
length = 1 + int(src[s-3])>>2
offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
case tagCopy4:
s += 5
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
return decodeErrCodeCorrupt
}
length = 1 + int(src[s-5])>>2
offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
}
if offset <= 0 || d < offset || length > len(dst)-d {
return decodeErrCodeCorrupt
}
// Copy from an earlier sub-slice of dst to a later sub-slice.
// If no overlap, use the built-in copy:
if offset >= length {
copy(dst[d:d+length], dst[d-offset:])
d += length
continue
}
// Unlike the built-in copy function, this byte-by-byte copy always runs
// forwards, even if the slices overlap. Conceptually, this is:
//
// d += forwardCopy(dst[d:d+length], dst[d-offset:])
//
// We align the slices into a and b and show the compiler they are the same size.
// This allows the loop to run without bounds checks.
a := dst[d : d+length]
b := dst[d-offset:]
b = b[:len(a)]
for i := range a {
a[i] = b[i]
}
d += length
}
if d != len(dst) {
return decodeErrCodeCorrupt
}
return 0
}

289
vendor/github.com/golang/snappy/encode.go generated vendored

@ -0,0 +1,289 @@
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package snappy
import (
"encoding/binary"
"errors"
"io"
)
// Encode returns the encoded form of src. The returned slice may be a sub-
// slice of dst if dst was large enough to hold the entire encoded block.
// Otherwise, a newly allocated slice will be returned.
//
// The dst and src must not overlap. It is valid to pass a nil dst.
//
// Encode handles the Snappy block format, not the Snappy stream format.
func Encode(dst, src []byte) []byte {
if n := MaxEncodedLen(len(src)); n < 0 {
panic(ErrTooLarge)
} else if len(dst) < n {
dst = make([]byte, n)
}
// The block starts with the varint-encoded length of the decompressed bytes.
d := binary.PutUvarint(dst, uint64(len(src)))
for len(src) > 0 {
p := src
src = nil
if len(p) > maxBlockSize {
p, src = p[:maxBlockSize], p[maxBlockSize:]
}
if len(p) < minNonLiteralBlockSize {
d += emitLiteral(dst[d:], p)
} else {
d += encodeBlock(dst[d:], p)
}
}
return dst[:d]
}
// inputMargin is the minimum number of extra input bytes to keep, inside
// encodeBlock's inner loop. On some architectures, this margin lets us
// implement a fast path for emitLiteral, where the copy of short (<= 16 byte)
// literals can be implemented as a single load to and store from a 16-byte
// register. That literal's actual length can be as short as 1 byte, so this
// can copy up to 15 bytes too much, but that's OK as subsequent iterations of
// the encoding loop will fix up the copy overrun, and this inputMargin ensures
// that we don't overrun the dst and src buffers.
const inputMargin = 16 - 1
// minNonLiteralBlockSize is the minimum size of the input to encodeBlock that
// could be encoded with a copy tag. This is the minimum with respect to the
// algorithm used by encodeBlock, not a minimum enforced by the file format.
//
// The encoded output must start with at least a 1 byte literal, as there are
// no previous bytes to copy. A minimal (1 byte) copy after that, generated
// from an emitCopy call in encodeBlock's main loop, would require at least
// another inputMargin bytes, for the reason above: we want any emitLiteral
// calls inside encodeBlock's main loop to use the fast path if possible, which
// requires being able to overrun by inputMargin bytes. Thus,
// minNonLiteralBlockSize equals 1 + 1 + inputMargin.
//
// The C++ code doesn't use this exact threshold, but it could, as discussed at
// https://groups.google.com/d/topic/snappy-compression/oGbhsdIJSJ8/discussion
// The difference between Go (2+inputMargin) and C++ (inputMargin) is purely an
// optimization. It should not affect the encoded form. This is tested by
// TestSameEncodingAsCppShortCopies.
const minNonLiteralBlockSize = 1 + 1 + inputMargin
// MaxEncodedLen returns the maximum length of a snappy block, given its
// uncompressed length.
//
// It will return a negative value if srcLen is too large to encode.
func MaxEncodedLen(srcLen int) int {
n := uint64(srcLen)
if n > 0xffffffff {
return -1
}
// Compressed data can be defined as:
// compressed := item* literal*
// item := literal* copy
//
// The trailing literal sequence has a space blowup of at most 62/60
// since a literal of length 60 needs one tag byte + one extra byte
// for length information.
//
// Item blowup is trickier to measure. Suppose the "copy" op copies
// 4 bytes of data. Because of a special check in the encoding code,
// we produce a 4-byte copy only if the offset is < 65536. Therefore
// the copy op takes 3 bytes to encode, and this type of item leads
// to at most the 62/60 blowup for representing literals.
//
// Suppose the "copy" op copies 5 bytes of data. If the offset is big
// enough, it will take 5 bytes to encode the copy op. Therefore the
// worst case here is a one-byte literal followed by a five-byte copy.
// That is, 6 bytes of input turn into 7 bytes of "compressed" data.
//
// This last factor dominates the blowup, so the final estimate is:
n = 32 + n + n/6
if n > 0xffffffff {
return -1
}
return int(n)
}
var errClosed = errors.New("snappy: Writer is closed")
// NewWriter returns a new Writer that compresses to w.
//
// The Writer returned does not buffer writes. There is no need to Flush or
// Close such a Writer.
//
// Deprecated: the Writer returned is not suitable for many small writes, only
// for few large writes. Use NewBufferedWriter instead, which is efficient
// regardless of the frequency and shape of the writes, and remember to Close
// that Writer when done.
func NewWriter(w io.Writer) *Writer {
return &Writer{
w: w,
obuf: make([]byte, obufLen),
}
}
// NewBufferedWriter returns a new Writer that compresses to w, using the
// framing format described at
// https://github.com/google/snappy/blob/master/framing_format.txt
//
// The Writer returned buffers writes. Users must call Close to guarantee all
// data has been forwarded to the underlying io.Writer. They may also call
// Flush zero or more times before calling Close.
func NewBufferedWriter(w io.Writer) *Writer {
return &Writer{
w: w,
ibuf: make([]byte, 0, maxBlockSize),
obuf: make([]byte, obufLen),
}
}
// Writer is an io.Writer that can write Snappy-compressed bytes.
//
// Writer handles the Snappy stream format, not the Snappy block format.
type Writer struct {
w io.Writer
err error
// ibuf is a buffer for the incoming (uncompressed) bytes.
//
// Its use is optional. For backwards compatibility, Writers created by the
// NewWriter function have ibuf == nil, do not buffer incoming bytes, and
// therefore do not need to be Flush'ed or Close'd.
ibuf []byte
// obuf is a buffer for the outgoing (compressed) bytes.
obuf []byte
// wroteStreamHeader is whether we have written the stream header.
wroteStreamHeader bool
}
// Reset discards the writer's state and switches the Snappy writer to write to
// w. This permits reusing a Writer rather than allocating a new one.
func (w *Writer) Reset(writer io.Writer) {
w.w = writer
w.err = nil
if w.ibuf != nil {
w.ibuf = w.ibuf[:0]
}
w.wroteStreamHeader = false
}
// Write satisfies the io.Writer interface.
func (w *Writer) Write(p []byte) (nRet int, errRet error) {
if w.ibuf == nil {
// Do not buffer incoming bytes. This does not perform or compress well
// if the caller of Writer.Write writes many small slices. This
// behavior is therefore deprecated, but still supported for backwards
// compatibility with code that doesn't explicitly Flush or Close.
return w.write(p)
}
// The remainder of this method is based on bufio.Writer.Write from the
// standard library.
for len(p) > (cap(w.ibuf)-len(w.ibuf)) && w.err == nil {
var n int
if len(w.ibuf) == 0 {
// Large write, empty buffer.
// Write directly from p to avoid copy.
n, _ = w.write(p)
} else {
n = copy(w.ibuf[len(w.ibuf):cap(w.ibuf)], p)
w.ibuf = w.ibuf[:len(w.ibuf)+n]
w.Flush()
}
nRet += n
p = p[n:]
}
if w.err != nil {
return nRet, w.err
}
n := copy(w.ibuf[len(w.ibuf):cap(w.ibuf)], p)
w.ibuf = w.ibuf[:len(w.ibuf)+n]
nRet += n
return nRet, nil
}
func (w *Writer) write(p []byte) (nRet int, errRet error) {
if w.err != nil {
return 0, w.err
}
for len(p) > 0 {
obufStart := len(magicChunk)
if !w.wroteStreamHeader {
w.wroteStreamHeader = true
copy(w.obuf, magicChunk)
obufStart = 0
}
var uncompressed []byte
if len(p) > maxBlockSize {
uncompressed, p = p[:maxBlockSize], p[maxBlockSize:]
} else {
uncompressed, p = p, nil
}
checksum := crc(uncompressed)
// Compress the buffer, discarding the result if the improvement
// isn't at least 12.5%.
compressed := Encode(w.obuf[obufHeaderLen:], uncompressed)
chunkType := uint8(chunkTypeCompressedData)
chunkLen := 4 + len(compressed)
obufEnd := obufHeaderLen + len(compressed)
if len(compressed) >= len(uncompressed)-len(uncompressed)/8 {
chunkType = chunkTypeUncompressedData
chunkLen = 4 + len(uncompressed)
obufEnd = obufHeaderLen
}
// Fill in the per-chunk header that comes before the body.
w.obuf[len(magicChunk)+0] = chunkType
w.obuf[len(magicChunk)+1] = uint8(chunkLen >> 0)
w.obuf[len(magicChunk)+2] = uint8(chunkLen >> 8)
w.obuf[len(magicChunk)+3] = uint8(chunkLen >> 16)
w.obuf[len(magicChunk)+4] = uint8(checksum >> 0)
w.obuf[len(magicChunk)+5] = uint8(checksum >> 8)
w.obuf[len(magicChunk)+6] = uint8(checksum >> 16)
w.obuf[len(magicChunk)+7] = uint8(checksum >> 24)
if _, err := w.w.Write(w.obuf[obufStart:obufEnd]); err != nil {
w.err = err
return nRet, err
}
if chunkType == chunkTypeUncompressedData {
if _, err := w.w.Write(uncompressed); err != nil {
w.err = err
return nRet, err
}
}
nRet += len(uncompressed)
}
return nRet, nil
}
// Flush flushes the Writer to its underlying io.Writer.
func (w *Writer) Flush() error {
if w.err != nil {
return w.err
}
if len(w.ibuf) == 0 {
return nil
}
w.write(w.ibuf)
w.ibuf = w.ibuf[:0]
return w.err
}
// Close calls Flush and then closes the Writer.
func (w *Writer) Close() error {
w.Flush()
ret := w.err
if w.err == nil {
w.err = errClosed
}
return ret
}

730
vendor/github.com/golang/snappy/encode_amd64.s generated vendored

@ -0,0 +1,730 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !appengine
// +build gc
// +build !noasm
#include "textflag.h"
// The XXX lines assemble on Go 1.4, 1.5 and 1.7, but not 1.6, due to a
// Go toolchain regression. See https://github.com/golang/go/issues/15426 and
// https://github.com/golang/snappy/issues/29
//
// As a workaround, the package was built with a known good assembler, and
// those instructions were disassembled by "objdump -d" to yield the
// 4e 0f b7 7c 5c 78 movzwq 0x78(%rsp,%r11,2),%r15
// style comments, in AT&T asm syntax. Note that rsp here is a physical
// register, not Go/asm's SP pseudo-register (see https://golang.org/doc/asm).
// The instructions were then encoded as "BYTE $0x.." sequences, which assemble
// fine on Go 1.6.
// The asm code generally follows the pure Go code in encode_other.go, except
// where marked with a "!!!".
// ----------------------------------------------------------------------------
// func emitLiteral(dst, lit []byte) int
//
// All local variables fit into registers. The register allocation:
// - AX len(lit)
// - BX n
// - DX return value
// - DI &dst[i]
// - R10 &lit[0]
//
// The 24 bytes of stack space is to call runtime·memmove.
//
// The unusual register allocation of local variables, such as R10 for the
// source pointer, matches the allocation used at the call site in encodeBlock,
// which makes it easier to manually inline this function.
TEXT ·emitLiteral(SB), NOSPLIT, $24-56
MOVQ dst_base+0(FP), DI
MOVQ lit_base+24(FP), R10
MOVQ lit_len+32(FP), AX
MOVQ AX, DX
MOVL AX, BX
SUBL $1, BX
CMPL BX, $60
JLT oneByte
CMPL BX, $256
JLT twoBytes
threeBytes:
MOVB $0xf4, 0(DI)
MOVW BX, 1(DI)
ADDQ $3, DI
ADDQ $3, DX
JMP memmove
twoBytes:
MOVB $0xf0, 0(DI)
MOVB BX, 1(DI)
ADDQ $2, DI
ADDQ $2, DX
JMP memmove
oneByte:
SHLB $2, BX
MOVB BX, 0(DI)
ADDQ $1, DI
ADDQ $1, DX
memmove:
MOVQ DX, ret+48(FP)
// copy(dst[i:], lit)
//
// This means calling runtime·memmove(&dst[i], &lit[0], len(lit)), so we push
// DI, R10 and AX as arguments.
MOVQ DI, 0(SP)
MOVQ R10, 8(SP)
MOVQ AX, 16(SP)
CALL runtime·memmove(SB)
RET
// ----------------------------------------------------------------------------
// func emitCopy(dst []byte, offset, length int) int
//
// All local variables fit into registers. The register allocation:
// - AX length
// - SI &dst[0]
// - DI &dst[i]
// - R11 offset
//
// The unusual register allocation of local variables, such as R11 for the
// offset, matches the allocation used at the call site in encodeBlock, which
// makes it easier to manually inline this function.
TEXT ·emitCopy(SB), NOSPLIT, $0-48
MOVQ dst_base+0(FP), DI
MOVQ DI, SI
MOVQ offset+24(FP), R11
MOVQ length+32(FP), AX
loop0:
// for length >= 68 { etc }
CMPL AX, $68
JLT step1
// Emit a length 64 copy, encoded as 3 bytes.
MOVB $0xfe, 0(DI)
MOVW R11, 1(DI)
ADDQ $3, DI
SUBL $64, AX
JMP loop0
step1:
// if length > 64 { etc }
CMPL AX, $64
JLE step2
// Emit a length 60 copy, encoded as 3 bytes.
MOVB $0xee, 0(DI)
MOVW R11, 1(DI)
ADDQ $3, DI
SUBL $60, AX
step2:
// if length >= 12 || offset >= 2048 { goto step3 }
CMPL AX, $12
JGE step3
CMPL R11, $2048
JGE step3
// Emit the remaining copy, encoded as 2 bytes.
MOVB R11, 1(DI)
SHRL $8, R11
SHLB $5, R11
SUBB $4, AX
SHLB $2, AX
ORB AX, R11
ORB $1, R11
MOVB R11, 0(DI)
ADDQ $2, DI
// Return the number of bytes written.
SUBQ SI, DI
MOVQ DI, ret+40(FP)
RET
step3:
// Emit the remaining copy, encoded as 3 bytes.
SUBL $1, AX
SHLB $2, AX
ORB $2, AX
MOVB AX, 0(DI)
MOVW R11, 1(DI)
ADDQ $3, DI
// Return the number of bytes written.
SUBQ SI, DI
MOVQ DI, ret+40(FP)
RET
// ----------------------------------------------------------------------------
// func extendMatch(src []byte, i, j int) int
//
// All local variables fit into registers. The register allocation:
// - DX &src[0]
// - SI &src[j]
// - R13 &src[len(src) - 8]
// - R14 &src[len(src)]
// - R15 &src[i]
//
// The unusual register allocation of local variables, such as R15 for a source
// pointer, matches the allocation used at the call site in encodeBlock, which
// makes it easier to manually inline this function.
TEXT ·extendMatch(SB), NOSPLIT, $0-48
MOVQ src_base+0(FP), DX
MOVQ src_len+8(FP), R14
MOVQ i+24(FP), R15
MOVQ j+32(FP), SI
ADDQ DX, R14
ADDQ DX, R15
ADDQ DX, SI
MOVQ R14, R13
SUBQ $8, R13
cmp8:
// As long as we are 8 or more bytes before the end of src, we can load and
// compare 8 bytes at a time. If those 8 bytes are equal, repeat.
CMPQ SI, R13
JA cmp1
MOVQ (R15), AX
MOVQ (SI), BX
CMPQ AX, BX
JNE bsf
ADDQ $8, R15
ADDQ $8, SI
JMP cmp8
bsf:
// If those 8 bytes were not equal, XOR the two 8 byte values, and return
// the index of the first byte that differs. The BSF instruction finds the
// least significant 1 bit, the amd64 architecture is little-endian, and
// the shift by 3 converts a bit index to a byte index.
XORQ AX, BX
BSFQ BX, BX
SHRQ $3, BX
ADDQ BX, SI
// Convert from &src[ret] to ret.
SUBQ DX, SI
MOVQ SI, ret+40(FP)
RET
cmp1:
// In src's tail, compare 1 byte at a time.
CMPQ SI, R14
JAE extendMatchEnd
MOVB (R15), AX
MOVB (SI), BX
CMPB AX, BX
JNE extendMatchEnd
ADDQ $1, R15
ADDQ $1, SI
JMP cmp1
extendMatchEnd:
// Convert from &src[ret] to ret.
SUBQ DX, SI
MOVQ SI, ret+40(FP)
RET
// ----------------------------------------------------------------------------
// func encodeBlock(dst, src []byte) (d int)
//
// All local variables fit into registers, other than "var table". The register
// allocation:
// - AX . .
// - BX . .
// - CX 56 shift (note that amd64 shifts by non-immediates must use CX).
// - DX 64 &src[0], tableSize
// - SI 72 &src[s]
// - DI 80 &dst[d]
// - R9 88 sLimit
// - R10 . &src[nextEmit]
// - R11 96 prevHash, currHash, nextHash, offset
// - R12 104 &src[base], skip
// - R13 . &src[nextS], &src[len(src) - 8]
// - R14 . len(src), bytesBetweenHashLookups, &src[len(src)], x
// - R15 112 candidate
//
// The second column (56, 64, etc) is the stack offset to spill the registers
// when calling other functions. We could pack this slightly tighter, but it's
// simpler to have a dedicated spill map independent of the function called.
//
// "var table [maxTableSize]uint16" takes up 32768 bytes of stack space. An
// extra 56 bytes, to call other functions, and an extra 64 bytes, to spill
// local variables (registers) during calls gives 32768 + 56 + 64 = 32888.
TEXT ·encodeBlock(SB), 0, $32888-56
MOVQ dst_base+0(FP), DI
MOVQ src_base+24(FP), SI
MOVQ src_len+32(FP), R14
// shift, tableSize := uint32(32-8), 1<<8
MOVQ $24, CX
MOVQ $256, DX
calcShift:
// for ; tableSize < maxTableSize && tableSize < len(src); tableSize *= 2 {
// shift--
// }
CMPQ DX, $16384
JGE varTable
CMPQ DX, R14
JGE varTable
SUBQ $1, CX
SHLQ $1, DX
JMP calcShift
varTable:
// var table [maxTableSize]uint16
//
// In the asm code, unlike the Go code, we can zero-initialize only the
// first tableSize elements. Each uint16 element is 2 bytes and each MOVOU
// writes 16 bytes, so we can do only tableSize/8 writes instead of the
// 2048 writes that would zero-initialize all of table's 32768 bytes.
SHRQ $3, DX
LEAQ table-32768(SP), BX
PXOR X0, X0
memclr:
MOVOU X0, 0(BX)
ADDQ $16, BX
SUBQ $1, DX
JNZ memclr
// !!! DX = &src[0]
MOVQ SI, DX
// sLimit := len(src) - inputMargin
MOVQ R14, R9
SUBQ $15, R9
// !!! Pre-emptively spill CX, DX and R9 to the stack. Their values don't
// change for the rest of the function.
MOVQ CX, 56(SP)
MOVQ DX, 64(SP)
MOVQ R9, 88(SP)
// nextEmit := 0
MOVQ DX, R10
// s := 1
ADDQ $1, SI
// nextHash := hash(load32(src, s), shift)
MOVL 0(SI), R11
IMULL $0x1e35a7bd, R11
SHRL CX, R11
outer:
// for { etc }
// skip := 32
MOVQ $32, R12
// nextS := s
MOVQ SI, R13
// candidate := 0
MOVQ $0, R15
inner0:
// for { etc }
// s := nextS
MOVQ R13, SI
// bytesBetweenHashLookups := skip >> 5
MOVQ R12, R14
SHRQ $5, R14
// nextS = s + bytesBetweenHashLookups
ADDQ R14, R13
// skip += bytesBetweenHashLookups
ADDQ R14, R12
// if nextS > sLimit { goto emitRemainder }
MOVQ R13, AX
SUBQ DX, AX
CMPQ AX, R9
JA emitRemainder
// candidate = int(table[nextHash])
// XXX: MOVWQZX table-32768(SP)(R11*2), R15
// XXX: 4e 0f b7 7c 5c 78 movzwq 0x78(%rsp,%r11,2),%r15
BYTE $0x4e
BYTE $0x0f
BYTE $0xb7
BYTE $0x7c
BYTE $0x5c
BYTE $0x78
// table[nextHash] = uint16(s)
MOVQ SI, AX
SUBQ DX, AX
// XXX: MOVW AX, table-32768(SP)(R11*2)
// XXX: 66 42 89 44 5c 78 mov %ax,0x78(%rsp,%r11,2)
BYTE $0x66
BYTE $0x42
BYTE $0x89
BYTE $0x44
BYTE $0x5c
BYTE $0x78
// nextHash = hash(load32(src, nextS), shift)
MOVL 0(R13), R11
IMULL $0x1e35a7bd, R11
SHRL CX, R11
// if load32(src, s) != load32(src, candidate) { continue } break
MOVL 0(SI), AX
MOVL (DX)(R15*1), BX
CMPL AX, BX
JNE inner0
fourByteMatch:
// As per the encode_other.go code:
//
// A 4-byte match has been found. We'll later see etc.
// !!! Jump to a fast path for short (<= 16 byte) literals. See the comment
// on inputMargin in encode.go.
MOVQ SI, AX
SUBQ R10, AX
CMPQ AX, $16
JLE emitLiteralFastPath
// ----------------------------------------
// Begin inline of the emitLiteral call.
//
// d += emitLiteral(dst[d:], src[nextEmit:s])
MOVL AX, BX
SUBL $1, BX
CMPL BX, $60
JLT inlineEmitLiteralOneByte
CMPL BX, $256
JLT inlineEmitLiteralTwoBytes
inlineEmitLiteralThreeBytes:
MOVB $0xf4, 0(DI)
MOVW BX, 1(DI)
ADDQ $3, DI
JMP inlineEmitLiteralMemmove
inlineEmitLiteralTwoBytes:
MOVB $0xf0, 0(DI)
MOVB BX, 1(DI)
ADDQ $2, DI
JMP inlineEmitLiteralMemmove
inlineEmitLiteralOneByte:
SHLB $2, BX
MOVB BX, 0(DI)
ADDQ $1, DI
inlineEmitLiteralMemmove:
// Spill local variables (registers) onto the stack; call; unspill.
//
// copy(dst[i:], lit)
//
// This means calling runtime·memmove(&dst[i], &lit[0], len(lit)), so we push
// DI, R10 and AX as arguments.
MOVQ DI, 0(SP)
MOVQ R10, 8(SP)
MOVQ AX, 16(SP)
ADDQ AX, DI // Finish the "d +=" part of "d += emitLiteral(etc)".
MOVQ SI, 72(SP)
MOVQ DI, 80(SP)
MOVQ R15, 112(SP)
CALL runtime·memmove(SB)
MOVQ 56(SP), CX
MOVQ 64(SP), DX
MOVQ 72(SP), SI
MOVQ 80(SP), DI
MOVQ 88(SP), R9
MOVQ 112(SP), R15
JMP inner1
inlineEmitLiteralEnd:
// End inline of the emitLiteral call.
// ----------------------------------------
emitLiteralFastPath:
// !!! Emit the 1-byte encoding "uint8(len(lit)-1)<<2".
MOVB AX, BX
SUBB $1, BX
SHLB $2, BX
MOVB BX, (DI)
ADDQ $1, DI
// !!! Implement the copy from lit to dst as a 16-byte load and store.
// (Encode's documentation says that dst and src must not overlap.)
//
// This always copies 16 bytes, instead of only len(lit) bytes, but that's
// OK. Subsequent iterations will fix up the overrun.
//
// Note that on amd64, it is legal and cheap to issue unaligned 8-byte or
// 16-byte loads and stores. This technique probably wouldn't be as
// effective on architectures that are fussier about alignment.
MOVOU 0(R10), X0
MOVOU X0, 0(DI)
ADDQ AX, DI
inner1:
// for { etc }
// base := s
MOVQ SI, R12
// !!! offset := base - candidate
MOVQ R12, R11
SUBQ R15, R11
SUBQ DX, R11
// ----------------------------------------
// Begin inline of the extendMatch call.
//
// s = extendMatch(src, candidate+4, s+4)
// !!! R14 = &src[len(src)]
MOVQ src_len+32(FP), R14
ADDQ DX, R14
// !!! R13 = &src[len(src) - 8]
MOVQ R14, R13
SUBQ $8, R13
// !!! R15 = &src[candidate + 4]
ADDQ $4, R15
ADDQ DX, R15
// !!! s += 4
ADDQ $4, SI
inlineExtendMatchCmp8:
// As long as we are 8 or more bytes before the end of src, we can load and
// compare 8 bytes at a time. If those 8 bytes are equal, repeat.
CMPQ SI, R13
JA inlineExtendMatchCmp1
MOVQ (R15), AX
MOVQ (SI), BX
CMPQ AX, BX
JNE inlineExtendMatchBSF
ADDQ $8, R15
ADDQ $8, SI
JMP inlineExtendMatchCmp8
inlineExtendMatchBSF:
// If those 8 bytes were not equal, XOR the two 8 byte values, and return
// the index of the first byte that differs. The BSF instruction finds the
// least significant 1 bit, the amd64 architecture is little-endian, and
// the shift by 3 converts a bit index to a byte index.
XORQ AX, BX
BSFQ BX, BX
SHRQ $3, BX
ADDQ BX, SI
JMP inlineExtendMatchEnd
inlineExtendMatchCmp1:
// In src's tail, compare 1 byte at a time.
CMPQ SI, R14
JAE inlineExtendMatchEnd
MOVB (R15), AX
MOVB (SI), BX
CMPB AX, BX
JNE inlineExtendMatchEnd
ADDQ $1, R15
ADDQ $1, SI
JMP inlineExtendMatchCmp1
inlineExtendMatchEnd:
// End inline of the extendMatch call.
// ----------------------------------------
// ----------------------------------------
// Begin inline of the emitCopy call.
//
// d += emitCopy(dst[d:], base-candidate, s-base)
// !!! length := s - base
MOVQ SI, AX
SUBQ R12, AX
inlineEmitCopyLoop0:
// for length >= 68 { etc }
CMPL AX, $68
JLT inlineEmitCopyStep1
// Emit a length 64 copy, encoded as 3 bytes.
MOVB $0xfe, 0(DI)
MOVW R11, 1(DI)
ADDQ $3, DI
SUBL $64, AX
JMP inlineEmitCopyLoop0
inlineEmitCopyStep1:
// if length > 64 { etc }
CMPL AX, $64
JLE inlineEmitCopyStep2
// Emit a length 60 copy, encoded as 3 bytes.
MOVB $0xee, 0(DI)
MOVW R11, 1(DI)
ADDQ $3, DI
SUBL $60, AX
inlineEmitCopyStep2:
// if length >= 12 || offset >= 2048 { goto inlineEmitCopyStep3 }
CMPL AX, $12
JGE inlineEmitCopyStep3
CMPL R11, $2048
JGE inlineEmitCopyStep3
// Emit the remaining copy, encoded as 2 bytes.
MOVB R11, 1(DI)
SHRL $8, R11
SHLB $5, R11
SUBB $4, AX
SHLB $2, AX
ORB AX, R11
ORB $1, R11
MOVB R11, 0(DI)
ADDQ $2, DI
JMP inlineEmitCopyEnd
inlineEmitCopyStep3:
// Emit the remaining copy, encoded as 3 bytes.
SUBL $1, AX
SHLB $2, AX
ORB $2, AX
MOVB AX, 0(DI)
MOVW R11, 1(DI)
ADDQ $3, DI
inlineEmitCopyEnd:
// End inline of the emitCopy call.
// ----------------------------------------
// nextEmit = s
MOVQ SI, R10
// if s >= sLimit { goto emitRemainder }
MOVQ SI, AX
SUBQ DX, AX
CMPQ AX, R9
JAE emitRemainder
// As per the encode_other.go code:
//
// We could immediately etc.
// x := load64(src, s-1)
MOVQ -1(SI), R14
// prevHash := hash(uint32(x>>0), shift)
MOVL R14, R11
IMULL $0x1e35a7bd, R11
SHRL CX, R11
// table[prevHash] = uint16(s-1)
MOVQ SI, AX
SUBQ DX, AX
SUBQ $1, AX
// XXX: MOVW AX, table-32768(SP)(R11*2)
// XXX: 66 42 89 44 5c 78 mov %ax,0x78(%rsp,%r11,2)
BYTE $0x66
BYTE $0x42
BYTE $0x89
BYTE $0x44
BYTE $0x5c
BYTE $0x78
// currHash := hash(uint32(x>>8), shift)
SHRQ $8, R14
MOVL R14, R11
IMULL $0x1e35a7bd, R11
SHRL CX, R11
// candidate = int(table[currHash])
// XXX: MOVWQZX table-32768(SP)(R11*2), R15
// XXX: 4e 0f b7 7c 5c 78 movzwq 0x78(%rsp,%r11,2),%r15
BYTE $0x4e
BYTE $0x0f
BYTE $0xb7
BYTE $0x7c
BYTE $0x5c
BYTE $0x78
// table[currHash] = uint16(s)
ADDQ $1, AX
// XXX: MOVW AX, table-32768(SP)(R11*2)
// XXX: 66 42 89 44 5c 78 mov %ax,0x78(%rsp,%r11,2)
BYTE $0x66
BYTE $0x42
BYTE $0x89
BYTE $0x44
BYTE $0x5c
BYTE $0x78
// if uint32(x>>8) == load32(src, candidate) { continue }
MOVL (DX)(R15*1), BX
CMPL R14, BX
JEQ inner1
// nextHash = hash(uint32(x>>16), shift)
SHRQ $8, R14
MOVL R14, R11
IMULL $0x1e35a7bd, R11
SHRL CX, R11
// s++
ADDQ $1, SI
// break out of the inner1 for loop, i.e. continue the outer loop.
JMP outer
emitRemainder:
// if nextEmit < len(src) { etc }
MOVQ src_len+32(FP), AX
ADDQ DX, AX
CMPQ R10, AX
JEQ encodeBlockEnd
// d += emitLiteral(dst[d:], src[nextEmit:])
//
// Push args.
MOVQ DI, 0(SP)
MOVQ $0, 8(SP) // Unnecessary, as the callee ignores it, but conservative.
MOVQ $0, 16(SP) // Unnecessary, as the callee ignores it, but conservative.
MOVQ R10, 24(SP)
SUBQ R10, AX
MOVQ AX, 32(SP)
MOVQ AX, 40(SP) // Unnecessary, as the callee ignores it, but conservative.
// Spill local variables (registers) onto the stack; call; unspill.
MOVQ DI, 80(SP)
CALL ·emitLiteral(SB)
MOVQ 80(SP), DI
// Finish the "d +=" part of "d += emitLiteral(etc)".
ADDQ 48(SP), DI
encodeBlockEnd:
MOVQ dst_base+0(FP), AX
SUBQ AX, DI
MOVQ DI, d+48(FP)
RET

722
vendor/github.com/golang/snappy/encode_arm64.s generated vendored

@ -0,0 +1,722 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !appengine
// +build gc
// +build !noasm
#include "textflag.h"
// The asm code generally follows the pure Go code in encode_other.go, except
// where marked with a "!!!".
// ----------------------------------------------------------------------------
// func emitLiteral(dst, lit []byte) int
//
// All local variables fit into registers. The register allocation:
// - R3 len(lit)
// - R4 n
// - R6 return value
// - R8 &dst[i]
// - R10 &lit[0]
//
// The 32 bytes of stack space is to call runtime·memmove.
//
// The unusual register allocation of local variables, such as R10 for the
// source pointer, matches the allocation used at the call site in encodeBlock,
// which makes it easier to manually inline this function.
TEXT ·emitLiteral(SB), NOSPLIT, $32-56
MOVD dst_base+0(FP), R8
MOVD lit_base+24(FP), R10
MOVD lit_len+32(FP), R3
MOVD R3, R6
MOVW R3, R4
SUBW $1, R4, R4
CMPW $60, R4
BLT oneByte
CMPW $256, R4
BLT twoBytes
threeBytes:
MOVD $0xf4, R2
MOVB R2, 0(R8)
MOVW R4, 1(R8)
ADD $3, R8, R8
ADD $3, R6, R6
B memmove
twoBytes:
MOVD $0xf0, R2
MOVB R2, 0(R8)
MOVB R4, 1(R8)
ADD $2, R8, R8
ADD $2, R6, R6
B memmove
oneByte:
LSLW $2, R4, R4
MOVB R4, 0(R8)
ADD $1, R8, R8
ADD $1, R6, R6
memmove:
MOVD R6, ret+48(FP)
// copy(dst[i:], lit)
//
// This means calling runtime·memmove(&dst[i], &lit[0], len(lit)), so we push
// R8, R10 and R3 as arguments.
MOVD R8, 8(RSP)
MOVD R10, 16(RSP)
MOVD R3, 24(RSP)
CALL runtime·memmove(SB)
RET
// ----------------------------------------------------------------------------
// func emitCopy(dst []byte, offset, length int) int
//
// All local variables fit into registers. The register allocation:
// - R3 length
// - R7 &dst[0]
// - R8 &dst[i]
// - R11 offset
//
// The unusual register allocation of local variables, such as R11 for the
// offset, matches the allocation used at the call site in encodeBlock, which
// makes it easier to manually inline this function.
TEXT ·emitCopy(SB), NOSPLIT, $0-48
MOVD dst_base+0(FP), R8
MOVD R8, R7
MOVD offset+24(FP), R11
MOVD length+32(FP), R3
loop0:
// for length >= 68 { etc }
CMPW $68, R3
BLT step1
// Emit a length 64 copy, encoded as 3 bytes.
MOVD $0xfe, R2
MOVB R2, 0(R8)
MOVW R11, 1(R8)
ADD $3, R8, R8
SUB $64, R3, R3
B loop0
step1:
// if length > 64 { etc }
CMP $64, R3
BLE step2
// Emit a length 60 copy, encoded as 3 bytes.
MOVD $0xee, R2
MOVB R2, 0(R8)
MOVW R11, 1(R8)
ADD $3, R8, R8
SUB $60, R3, R3
step2:
// if length >= 12 || offset >= 2048 { goto step3 }
CMP $12, R3
BGE step3
CMPW $2048, R11
BGE step3
// Emit the remaining copy, encoded as 2 bytes.
MOVB R11, 1(R8)
LSRW $3, R11, R11
AND $0xe0, R11, R11
SUB $4, R3, R3
LSLW $2, R3
AND $0xff, R3, R3
ORRW R3, R11, R11
ORRW $1, R11, R11
MOVB R11, 0(R8)
ADD $2, R8, R8
// Return the number of bytes written.
SUB R7, R8, R8
MOVD R8, ret+40(FP)
RET
step3:
// Emit the remaining copy, encoded as 3 bytes.
SUB $1, R3, R3
AND $0xff, R3, R3
LSLW $2, R3, R3
ORRW $2, R3, R3
MOVB R3, 0(R8)
MOVW R11, 1(R8)
ADD $3, R8, R8
// Return the number of bytes written.
SUB R7, R8, R8
MOVD R8, ret+40(FP)
RET
// ----------------------------------------------------------------------------
// func extendMatch(src []byte, i, j int) int
//
// All local variables fit into registers. The register allocation:
// - R6 &src[0]
// - R7 &src[j]
// - R13 &src[len(src) - 8]
// - R14 &src[len(src)]
// - R15 &src[i]
//
// The unusual register allocation of local variables, such as R15 for a source
// pointer, matches the allocation used at the call site in encodeBlock, which
// makes it easier to manually inline this function.
TEXT ·extendMatch(SB), NOSPLIT, $0-48
MOVD src_base+0(FP), R6
MOVD src_len+8(FP), R14
MOVD i+24(FP), R15
MOVD j+32(FP), R7
ADD R6, R14, R14
ADD R6, R15, R15
ADD R6, R7, R7
MOVD R14, R13
SUB $8, R13, R13
cmp8:
// As long as we are 8 or more bytes before the end of src, we can load and
// compare 8 bytes at a time. If those 8 bytes are equal, repeat.
CMP R13, R7
BHI cmp1
MOVD (R15), R3
MOVD (R7), R4
CMP R4, R3
BNE bsf
ADD $8, R15, R15
ADD $8, R7, R7
B cmp8
bsf:
// If those 8 bytes were not equal, XOR the two 8 byte values, and return
// the index of the first byte that differs.
// RBIT reverses the bit order, then CLZ counts the leading zeros, the
// combination of which finds the least significant bit which is set.
// The arm64 architecture is little-endian, and the shift by 3 converts
// a bit index to a byte index.
EOR R3, R4, R4
RBIT R4, R4
CLZ R4, R4
ADD R4>>3, R7, R7
// Convert from &src[ret] to ret.
SUB R6, R7, R7
MOVD R7, ret+40(FP)
RET
cmp1:
// In src's tail, compare 1 byte at a time.
CMP R7, R14
BLS extendMatchEnd
MOVB (R15), R3
MOVB (R7), R4
CMP R4, R3
BNE extendMatchEnd
ADD $1, R15, R15
ADD $1, R7, R7
B cmp1
extendMatchEnd:
// Convert from &src[ret] to ret.
SUB R6, R7, R7
MOVD R7, ret+40(FP)
RET
// ----------------------------------------------------------------------------
// func encodeBlock(dst, src []byte) (d int)
//
// All local variables fit into registers, other than "var table". The register
// allocation:
// - R3 . .
// - R4 . .
// - R5 64 shift
// - R6 72 &src[0], tableSize
// - R7 80 &src[s]
// - R8 88 &dst[d]
// - R9 96 sLimit
// - R10 . &src[nextEmit]
// - R11 104 prevHash, currHash, nextHash, offset
// - R12 112 &src[base], skip
// - R13 . &src[nextS], &src[len(src) - 8]
// - R14 . len(src), bytesBetweenHashLookups, &src[len(src)], x
// - R15 120 candidate
// - R16 . hash constant, 0x1e35a7bd
// - R17 . &table
// - . 128 table
//
// The second column (64, 72, etc) is the stack offset to spill the registers
// when calling other functions. We could pack this slightly tighter, but it's
// simpler to have a dedicated spill map independent of the function called.
//
// "var table [maxTableSize]uint16" takes up 32768 bytes of stack space. An
// extra 64 bytes, to call other functions, and an extra 64 bytes, to spill
// local variables (registers) during calls gives 32768 + 64 + 64 = 32896.
TEXT ·encodeBlock(SB), 0, $32896-56
MOVD dst_base+0(FP), R8
MOVD src_base+24(FP), R7
MOVD src_len+32(FP), R14
// shift, tableSize := uint32(32-8), 1<<8
MOVD $24, R5
MOVD $256, R6
MOVW $0xa7bd, R16
MOVKW $(0x1e35<<16), R16
calcShift:
// for ; tableSize < maxTableSize && tableSize < len(src); tableSize *= 2 {
// shift--
// }
MOVD $16384, R2
CMP R2, R6
BGE varTable
CMP R14, R6
BGE varTable
SUB $1, R5, R5
LSL $1, R6, R6
B calcShift
varTable:
// var table [maxTableSize]uint16
//
// In the asm code, unlike the Go code, we can zero-initialize only the
// first tableSize elements. Each uint16 element is 2 bytes and each
// iterations writes 64 bytes, so we can do only tableSize/32 writes
// instead of the 2048 writes that would zero-initialize all of table's
// 32768 bytes. This clear could overrun the first tableSize elements, but
// it won't overrun the allocated stack size.
ADD $128, RSP, R17
MOVD R17, R4
// !!! R6 = &src[tableSize]
ADD R6<<1, R17, R6
memclr:
STP.P (ZR, ZR), 64(R4)
STP (ZR, ZR), -48(R4)
STP (ZR, ZR), -32(R4)
STP (ZR, ZR), -16(R4)
CMP R4, R6
BHI memclr
// !!! R6 = &src[0]
MOVD R7, R6
// sLimit := len(src) - inputMargin
MOVD R14, R9
SUB $15, R9, R9
// !!! Pre-emptively spill R5, R6 and R9 to the stack. Their values don't
// change for the rest of the function.
MOVD R5, 64(RSP)
MOVD R6, 72(RSP)
MOVD R9, 96(RSP)
// nextEmit := 0
MOVD R6, R10
// s := 1
ADD $1, R7, R7
// nextHash := hash(load32(src, s), shift)
MOVW 0(R7), R11
MULW R16, R11, R11
LSRW R5, R11, R11
outer:
// for { etc }
// skip := 32
MOVD $32, R12
// nextS := s
MOVD R7, R13
// candidate := 0
MOVD $0, R15
inner0:
// for { etc }
// s := nextS
MOVD R13, R7
// bytesBetweenHashLookups := skip >> 5
MOVD R12, R14
LSR $5, R14, R14
// nextS = s + bytesBetweenHashLookups
ADD R14, R13, R13
// skip += bytesBetweenHashLookups
ADD R14, R12, R12
// if nextS > sLimit { goto emitRemainder }
MOVD R13, R3
SUB R6, R3, R3
CMP R9, R3
BHI emitRemainder
// candidate = int(table[nextHash])
MOVHU 0(R17)(R11<<1), R15
// table[nextHash] = uint16(s)
MOVD R7, R3
SUB R6, R3, R3
MOVH R3, 0(R17)(R11<<1)
// nextHash = hash(load32(src, nextS), shift)
MOVW 0(R13), R11
MULW R16, R11
LSRW R5, R11, R11
// if load32(src, s) != load32(src, candidate) { continue } break
MOVW 0(R7), R3
MOVW (R6)(R15), R4
CMPW R4, R3
BNE inner0
fourByteMatch:
// As per the encode_other.go code:
//
// A 4-byte match has been found. We'll later see etc.
// !!! Jump to a fast path for short (<= 16 byte) literals. See the comment
// on inputMargin in encode.go.
MOVD R7, R3
SUB R10, R3, R3
CMP $16, R3
BLE emitLiteralFastPath
// ----------------------------------------
// Begin inline of the emitLiteral call.
//
// d += emitLiteral(dst[d:], src[nextEmit:s])
MOVW R3, R4
SUBW $1, R4, R4
MOVW $60, R2
CMPW R2, R4
BLT inlineEmitLiteralOneByte
MOVW $256, R2
CMPW R2, R4
BLT inlineEmitLiteralTwoBytes
inlineEmitLiteralThreeBytes:
MOVD $0xf4, R1
MOVB R1, 0(R8)
MOVW R4, 1(R8)
ADD $3, R8, R8
B inlineEmitLiteralMemmove
inlineEmitLiteralTwoBytes:
MOVD $0xf0, R1
MOVB R1, 0(R8)
MOVB R4, 1(R8)
ADD $2, R8, R8
B inlineEmitLiteralMemmove
inlineEmitLiteralOneByte:
LSLW $2, R4, R4
MOVB R4, 0(R8)
ADD $1, R8, R8
inlineEmitLiteralMemmove:
// Spill local variables (registers) onto the stack; call; unspill.
//
// copy(dst[i:], lit)
//
// This means calling runtime·memmove(&dst[i], &lit[0], len(lit)), so we push
// R8, R10 and R3 as arguments.
MOVD R8, 8(RSP)
MOVD R10, 16(RSP)
MOVD R3, 24(RSP)
// Finish the "d +=" part of "d += emitLiteral(etc)".
ADD R3, R8, R8
MOVD R7, 80(RSP)
MOVD R8, 88(RSP)
MOVD R15, 120(RSP)
CALL runtime·memmove(SB)
MOVD 64(RSP), R5
MOVD 72(RSP), R6
MOVD 80(RSP), R7
MOVD 88(RSP), R8
MOVD 96(RSP), R9
MOVD 120(RSP), R15
ADD $128, RSP, R17
MOVW $0xa7bd, R16
MOVKW $(0x1e35<<16), R16
B inner1
inlineEmitLiteralEnd:
// End inline of the emitLiteral call.
// ----------------------------------------
emitLiteralFastPath:
// !!! Emit the 1-byte encoding "uint8(len(lit)-1)<<2".
MOVB R3, R4
SUBW $1, R4, R4
AND $0xff, R4, R4
LSLW $2, R4, R4
MOVB R4, (R8)
ADD $1, R8, R8
// !!! Implement the copy from lit to dst as a 16-byte load and store.
// (Encode's documentation says that dst and src must not overlap.)
//
// This always copies 16 bytes, instead of only len(lit) bytes, but that's
// OK. Subsequent iterations will fix up the overrun.
//
// Note that on arm64, it is legal and cheap to issue unaligned 8-byte or
// 16-byte loads and stores. This technique probably wouldn't be as
// effective on architectures that are fussier about alignment.
LDP 0(R10), (R0, R1)
STP (R0, R1), 0(R8)
ADD R3, R8, R8
inner1:
// for { etc }
// base := s
MOVD R7, R12
// !!! offset := base - candidate
MOVD R12, R11
SUB R15, R11, R11
SUB R6, R11, R11
// ----------------------------------------
// Begin inline of the extendMatch call.
//
// s = extendMatch(src, candidate+4, s+4)
// !!! R14 = &src[len(src)]
MOVD src_len+32(FP), R14
ADD R6, R14, R14
// !!! R13 = &src[len(src) - 8]
MOVD R14, R13
SUB $8, R13, R13
// !!! R15 = &src[candidate + 4]
ADD $4, R15, R15
ADD R6, R15, R15
// !!! s += 4
ADD $4, R7, R7
inlineExtendMatchCmp8:
// As long as we are 8 or more bytes before the end of src, we can load and
// compare 8 bytes at a time. If those 8 bytes are equal, repeat.
CMP R13, R7
BHI inlineExtendMatchCmp1
MOVD (R15), R3
MOVD (R7), R4
CMP R4, R3
BNE inlineExtendMatchBSF
ADD $8, R15, R15
ADD $8, R7, R7
B inlineExtendMatchCmp8
inlineExtendMatchBSF:
// If those 8 bytes were not equal, XOR the two 8 byte values, and return
// the index of the first byte that differs.
// RBIT reverses the bit order, then CLZ counts the leading zeros, the
// combination of which finds the least significant bit which is set.
// The arm64 architecture is little-endian, and the shift by 3 converts
// a bit index to a byte index.
EOR R3, R4, R4
RBIT R4, R4
CLZ R4, R4
ADD R4>>3, R7, R7
B inlineExtendMatchEnd
inlineExtendMatchCmp1:
// In src's tail, compare 1 byte at a time.
CMP R7, R14
BLS inlineExtendMatchEnd
MOVB (R15), R3
MOVB (R7), R4
CMP R4, R3
BNE inlineExtendMatchEnd
ADD $1, R15, R15
ADD $1, R7, R7
B inlineExtendMatchCmp1
inlineExtendMatchEnd:
// End inline of the extendMatch call.
// ----------------------------------------
// ----------------------------------------
// Begin inline of the emitCopy call.
//
// d += emitCopy(dst[d:], base-candidate, s-base)
// !!! length := s - base
MOVD R7, R3
SUB R12, R3, R3
inlineEmitCopyLoop0:
// for length >= 68 { etc }
MOVW $68, R2
CMPW R2, R3
BLT inlineEmitCopyStep1
// Emit a length 64 copy, encoded as 3 bytes.
MOVD $0xfe, R1
MOVB R1, 0(R8)
MOVW R11, 1(R8)
ADD $3, R8, R8
SUBW $64, R3, R3
B inlineEmitCopyLoop0
inlineEmitCopyStep1:
// if length > 64 { etc }
MOVW $64, R2
CMPW R2, R3
BLE inlineEmitCopyStep2
// Emit a length 60 copy, encoded as 3 bytes.
MOVD $0xee, R1
MOVB R1, 0(R8)
MOVW R11, 1(R8)
ADD $3, R8, R8
SUBW $60, R3, R3
inlineEmitCopyStep2:
// if length >= 12 || offset >= 2048 { goto inlineEmitCopyStep3 }
MOVW $12, R2
CMPW R2, R3
BGE inlineEmitCopyStep3
MOVW $2048, R2
CMPW R2, R11
BGE inlineEmitCopyStep3
// Emit the remaining copy, encoded as 2 bytes.
MOVB R11, 1(R8)
LSRW $8, R11, R11
LSLW $5, R11, R11
SUBW $4, R3, R3
AND $0xff, R3, R3
LSLW $2, R3, R3
ORRW R3, R11, R11
ORRW $1, R11, R11
MOVB R11, 0(R8)
ADD $2, R8, R8
B inlineEmitCopyEnd
inlineEmitCopyStep3:
// Emit the remaining copy, encoded as 3 bytes.
SUBW $1, R3, R3
LSLW $2, R3, R3
ORRW $2, R3, R3
MOVB R3, 0(R8)
MOVW R11, 1(R8)
ADD $3, R8, R8
inlineEmitCopyEnd:
// End inline of the emitCopy call.
// ----------------------------------------
// nextEmit = s
MOVD R7, R10
// if s >= sLimit { goto emitRemainder }
MOVD R7, R3
SUB R6, R3, R3
CMP R3, R9
BLS emitRemainder
// As per the encode_other.go code:
//
// We could immediately etc.
// x := load64(src, s-1)
MOVD -1(R7), R14
// prevHash := hash(uint32(x>>0), shift)
MOVW R14, R11
MULW R16, R11, R11
LSRW R5, R11, R11
// table[prevHash] = uint16(s-1)
MOVD R7, R3
SUB R6, R3, R3
SUB $1, R3, R3
MOVHU R3, 0(R17)(R11<<1)
// currHash := hash(uint32(x>>8), shift)
LSR $8, R14, R14
MOVW R14, R11
MULW R16, R11, R11
LSRW R5, R11, R11
// candidate = int(table[currHash])
MOVHU 0(R17)(R11<<1), R15
// table[currHash] = uint16(s)
ADD $1, R3, R3
MOVHU R3, 0(R17)(R11<<1)
// if uint32(x>>8) == load32(src, candidate) { continue }
MOVW (R6)(R15), R4
CMPW R4, R14
BEQ inner1
// nextHash = hash(uint32(x>>16), shift)
LSR $8, R14, R14
MOVW R14, R11
MULW R16, R11, R11
LSRW R5, R11, R11
// s++
ADD $1, R7, R7
// break out of the inner1 for loop, i.e. continue the outer loop.
B outer
emitRemainder:
// if nextEmit < len(src) { etc }
MOVD src_len+32(FP), R3
ADD R6, R3, R3
CMP R3, R10
BEQ encodeBlockEnd
// d += emitLiteral(dst[d:], src[nextEmit:])
//
// Push args.
MOVD R8, 8(RSP)
MOVD $0, 16(RSP) // Unnecessary, as the callee ignores it, but conservative.
MOVD $0, 24(RSP) // Unnecessary, as the callee ignores it, but conservative.
MOVD R10, 32(RSP)
SUB R10, R3, R3
MOVD R3, 40(RSP)
MOVD R3, 48(RSP) // Unnecessary, as the callee ignores it, but conservative.
// Spill local variables (registers) onto the stack; call; unspill.
MOVD R8, 88(RSP)
CALL ·emitLiteral(SB)
MOVD 88(RSP), R8
// Finish the "d +=" part of "d += emitLiteral(etc)".
MOVD 56(RSP), R1
ADD R1, R8, R8
encodeBlockEnd:
MOVD dst_base+0(FP), R3
SUB R3, R8, R8
MOVD R8, d+48(FP)
RET

30
vendor/github.com/golang/snappy/encode_asm.go generated vendored

@ -0,0 +1,30 @@
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !appengine
// +build gc
// +build !noasm
// +build amd64 arm64
package snappy
// emitLiteral has the same semantics as in encode_other.go.
//
//go:noescape
func emitLiteral(dst, lit []byte) int
// emitCopy has the same semantics as in encode_other.go.
//
//go:noescape
func emitCopy(dst []byte, offset, length int) int
// extendMatch has the same semantics as in encode_other.go.
//
//go:noescape
func extendMatch(src []byte, i, j int) int
// encodeBlock has the same semantics as in encode_other.go.
//
//go:noescape
func encodeBlock(dst, src []byte) (d int)

238
vendor/github.com/golang/snappy/encode_other.go generated vendored

@ -0,0 +1,238 @@
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !amd64,!arm64 appengine !gc noasm
package snappy
func load32(b []byte, i int) uint32 {
b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line.
return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
}
func load64(b []byte, i int) uint64 {
b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line.
return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
}
// emitLiteral writes a literal chunk and returns the number of bytes written.
//
// It assumes that:
// dst is long enough to hold the encoded bytes
// 1 <= len(lit) && len(lit) <= 65536
func emitLiteral(dst, lit []byte) int {
i, n := 0, uint(len(lit)-1)
switch {
case n < 60:
dst[0] = uint8(n)<<2 | tagLiteral
i = 1
case n < 1<<8:
dst[0] = 60<<2 | tagLiteral
dst[1] = uint8(n)
i = 2
default:
dst[0] = 61<<2 | tagLiteral
dst[1] = uint8(n)
dst[2] = uint8(n >> 8)
i = 3
}
return i + copy(dst[i:], lit)
}
// emitCopy writes a copy chunk and returns the number of bytes written.
//
// It assumes that:
// dst is long enough to hold the encoded bytes
// 1 <= offset && offset <= 65535
// 4 <= length && length <= 65535
func emitCopy(dst []byte, offset, length int) int {
i := 0
// The maximum length for a single tagCopy1 or tagCopy2 op is 64 bytes. The
// threshold for this loop is a little higher (at 68 = 64 + 4), and the
// length emitted down below is is a little lower (at 60 = 64 - 4), because
// it's shorter to encode a length 67 copy as a length 60 tagCopy2 followed
// by a length 7 tagCopy1 (which encodes as 3+2 bytes) than to encode it as
// a length 64 tagCopy2 followed by a length 3 tagCopy2 (which encodes as
// 3+3 bytes). The magic 4 in the 64±4 is because the minimum length for a
// tagCopy1 op is 4 bytes, which is why a length 3 copy has to be an
// encodes-as-3-bytes tagCopy2 instead of an encodes-as-2-bytes tagCopy1.
for length >= 68 {
// Emit a length 64 copy, encoded as 3 bytes.
dst[i+0] = 63<<2 | tagCopy2
dst[i+1] = uint8(offset)
dst[i+2] = uint8(offset >> 8)
i += 3
length -= 64
}
if length > 64 {
// Emit a length 60 copy, encoded as 3 bytes.
dst[i+0] = 59<<2 | tagCopy2
dst[i+1] = uint8(offset)
dst[i+2] = uint8(offset >> 8)
i += 3
length -= 60
}
if length >= 12 || offset >= 2048 {
// Emit the remaining copy, encoded as 3 bytes.
dst[i+0] = uint8(length-1)<<2 | tagCopy2
dst[i+1] = uint8(offset)
dst[i+2] = uint8(offset >> 8)
return i + 3
}
// Emit the remaining copy, encoded as 2 bytes.
dst[i+0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
dst[i+1] = uint8(offset)
return i + 2
}
// extendMatch returns the largest k such that k <= len(src) and that
// src[i:i+k-j] and src[j:k] have the same contents.
//
// It assumes that:
// 0 <= i && i < j && j <= len(src)
func extendMatch(src []byte, i, j int) int {
for ; j < len(src) && src[i] == src[j]; i, j = i+1, j+1 {
}
return j
}
func hash(u, shift uint32) uint32 {
return (u * 0x1e35a7bd) >> shift
}
// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It
// assumes that the varint-encoded length of the decompressed bytes has already
// been written.
//
// It also assumes that:
// len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlock(dst, src []byte) (d int) {
// Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive.
// The table element type is uint16, as s < sLimit and sLimit < len(src)
// and len(src) <= maxBlockSize and maxBlockSize == 65536.
const (
maxTableSize = 1 << 14
// tableMask is redundant, but helps the compiler eliminate bounds
// checks.
tableMask = maxTableSize - 1
)
shift := uint32(32 - 8)
for tableSize := 1 << 8; tableSize < maxTableSize && tableSize < len(src); tableSize *= 2 {
shift--
}
// In Go, all array elements are zero-initialized, so there is no advantage
// to a smaller tableSize per se. However, it matches the C++ algorithm,
// and in the asm versions of this code, we can get away with zeroing only
// the first tableSize elements.
var table [maxTableSize]uint16
// sLimit is when to stop looking for offset/length copies. The inputMargin
// lets us use a fast path for emitLiteral in the main loop, while we are
// looking for copies.
sLimit := len(src) - inputMargin
// nextEmit is where in src the next emitLiteral should start from.
nextEmit := 0
// The encoded form must start with a literal, as there are no previous
// bytes to copy, so we start looking for hash matches at s == 1.
s := 1
nextHash := hash(load32(src, s), shift)
for {
// Copied from the C++ snappy implementation:
//
// Heuristic match skipping: If 32 bytes are scanned with no matches
// found, start looking only at every other byte. If 32 more bytes are
// scanned (or skipped), look at every third byte, etc.. When a match
// is found, immediately go back to looking at every byte. This is a
// small loss (~5% performance, ~0.1% density) for compressible data
// due to more bookkeeping, but for non-compressible data (such as
// JPEG) it's a huge win since the compressor quickly "realizes" the
// data is incompressible and doesn't bother looking for matches
// everywhere.
//
// The "skip" variable keeps track of how many bytes there are since
// the last match; dividing it by 32 (ie. right-shifting by five) gives
// the number of bytes to move ahead for each iteration.
skip := 32
nextS := s
candidate := 0
for {
s = nextS
bytesBetweenHashLookups := skip >> 5
nextS = s + bytesBetweenHashLookups
skip += bytesBetweenHashLookups
if nextS > sLimit {
goto emitRemainder
}
candidate = int(table[nextHash&tableMask])
table[nextHash&tableMask] = uint16(s)
nextHash = hash(load32(src, nextS), shift)
if load32(src, s) == load32(src, candidate) {
break
}
}
// A 4-byte match has been found. We'll later see if more than 4 bytes
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
// them as literal bytes.
d += emitLiteral(dst[d:], src[nextEmit:s])
// Call emitCopy, and then see if another emitCopy could be our next
// move. Repeat until we find no match for the input immediately after
// what was consumed by the last emitCopy call.
//
// If we exit this loop normally then we need to call emitLiteral next,
// though we don't yet know how big the literal will be. We handle that
// by proceeding to the next iteration of the main loop. We also can
// exit this loop via goto if we get close to exhausting the input.
for {
// Invariant: we have a 4-byte match at s, and no need to emit any
// literal bytes prior to s.
base := s
// Extend the 4-byte match as long as possible.
//
// This is an inlined version of:
// s = extendMatch(src, candidate+4, s+4)
s += 4
for i := candidate + 4; s < len(src) && src[i] == src[s]; i, s = i+1, s+1 {
}
d += emitCopy(dst[d:], base-candidate, s-base)
nextEmit = s
if s >= sLimit {
goto emitRemainder
}
// We could immediately start working at s now, but to improve
// compression we first update the hash table at s-1 and at s. If
// another emitCopy is not our next move, also calculate nextHash
// at s+1. At least on GOARCH=amd64, these three hash calculations
// are faster as one load64 call (with some shifts) instead of
// three load32 calls.
x := load64(src, s-1)
prevHash := hash(uint32(x>>0), shift)
table[prevHash&tableMask] = uint16(s - 1)
currHash := hash(uint32(x>>8), shift)
candidate = int(table[currHash&tableMask])
table[currHash&tableMask] = uint16(s)
if uint32(x>>8) != load32(src, candidate) {
nextHash = hash(uint32(x>>16), shift)
s++
break
}
}
}
emitRemainder:
if nextEmit < len(src) {
d += emitLiteral(dst[d:], src[nextEmit:])
}
return d
}

98
vendor/github.com/golang/snappy/snappy.go generated vendored

@ -0,0 +1,98 @@
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package snappy implements the Snappy compression format. It aims for very
// high speeds and reasonable compression.
//
// There are actually two Snappy formats: block and stream. They are related,
// but different: trying to decompress block-compressed data as a Snappy stream
// will fail, and vice versa. The block format is the Decode and Encode
// functions and the stream format is the Reader and Writer types.
//
// The block format, the more common case, is used when the complete size (the
// number of bytes) of the original data is known upfront, at the time
// compression starts. The stream format, also known as the framing format, is
// for when that isn't always true.
//
// The canonical, C++ implementation is at https://github.com/google/snappy and
// it only implements the block format.
package snappy // import "github.com/golang/snappy"
import (
"hash/crc32"
)
/*
Each encoded block begins with the varint-encoded length of the decoded data,
followed by a sequence of chunks. Chunks begin and end on byte boundaries. The
first byte of each chunk is broken into its 2 least and 6 most significant bits
called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag.
Zero means a literal tag. All other values mean a copy tag.
For literal tags:
- If m < 60, the next 1 + m bytes are literal bytes.
- Otherwise, let n be the little-endian unsigned integer denoted by the next
m - 59 bytes. The next 1 + n bytes after that are literal bytes.
For copy tags, length bytes are copied from offset bytes ago, in the style of
Lempel-Ziv compression algorithms. In particular:
- For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12).
The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10
of the offset. The next byte is bits 0-7 of the offset.
- For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65).
The length is 1 + m. The offset is the little-endian unsigned integer
denoted by the next 2 bytes.
- For l == 3, this tag is a legacy format that is no longer issued by most
encoders. Nonetheless, the offset ranges in [0, 1<<32) and the length in
[1, 65). The length is 1 + m. The offset is the little-endian unsigned
integer denoted by the next 4 bytes.
*/
const (
tagLiteral = 0x00
tagCopy1 = 0x01
tagCopy2 = 0x02
tagCopy4 = 0x03
)
const (
checksumSize = 4
chunkHeaderSize = 4
magicChunk = "\xff\x06\x00\x00" + magicBody
magicBody = "sNaPpY"
// maxBlockSize is the maximum size of the input to encodeBlock. It is not
// part of the wire format per se, but some parts of the encoder assume
// that an offset fits into a uint16.
//
// Also, for the framing format (Writer type instead of Encode function),
// https://github.com/google/snappy/blob/master/framing_format.txt says
// that "the uncompressed data in a chunk must be no longer than 65536
// bytes".
maxBlockSize = 65536
// maxEncodedLenOfMaxBlockSize equals MaxEncodedLen(maxBlockSize), but is
// hard coded to be a const instead of a variable, so that obufLen can also
// be a const. Their equivalence is confirmed by
// TestMaxEncodedLenOfMaxBlockSize.
maxEncodedLenOfMaxBlockSize = 76490
obufHeaderLen = len(magicChunk) + checksumSize + chunkHeaderSize
obufLen = obufHeaderLen + maxEncodedLenOfMaxBlockSize
)
const (
chunkTypeCompressedData = 0x00
chunkTypeUncompressedData = 0x01
chunkTypePadding = 0xfe
chunkTypeStreamIdentifier = 0xff
)
var crcTable = crc32.MakeTable(crc32.Castagnoli)
// crc implements the checksum specified in section 3 of
// https://github.com/google/snappy/blob/master/framing_format.txt
func crc(b []byte) uint32 {
c := crc32.Update(0, crcTable, b)
return uint32(c>>15|c<<17) + 0xa282ead8
}

9
vendor/github.com/google/uuid/.travis.yml generated vendored

@ -0,0 +1,9 @@
language: go
go:
- 1.4.3
- 1.5.3
- tip
script:
- go test -v ./...

10
vendor/github.com/google/uuid/CONTRIBUTING.md generated vendored

@ -0,0 +1,10 @@
# How to contribute
We definitely welcome patches and contribution to this project!
### Legal requirements
In order to protect both you and ourselves, you will need to sign the
[Contributor License Agreement](https://cla.developers.google.com/clas).
You may have already signed it for other Google projects.

9
vendor/github.com/google/uuid/CONTRIBUTORS generated vendored

@ -0,0 +1,9 @@
Paul Borman <borman@google.com>
bmatsuo
shawnps
theory
jboverfelt
dsymonds
cd1
wallclockbuilder
dansouza

27
vendor/github.com/google/uuid/LICENSE generated vendored

@ -0,0 +1,27 @@
Copyright (c) 2009,2014 Google Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

19
vendor/github.com/google/uuid/README.md generated vendored

@ -0,0 +1,19 @@
# uuid ![build status](https://travis-ci.org/google/uuid.svg?branch=master)
The uuid package generates and inspects UUIDs based on
[RFC 4122](http://tools.ietf.org/html/rfc4122)
and DCE 1.1: Authentication and Security Services.
This package is based on the github.com/pborman/uuid package (previously named
code.google.com/p/go-uuid). It differs from these earlier packages in that
a UUID is a 16 byte array rather than a byte slice. One loss due to this
change is the ability to represent an invalid UUID (vs a NIL UUID).
###### Install
`go get github.com/google/uuid`
###### Documentation
[![GoDoc](https://godoc.org/github.com/google/uuid?status.svg)](http://godoc.org/github.com/google/uuid)
Full `go doc` style documentation for the package can be viewed online without
installing this package by using the GoDoc site here:
http://pkg.go.dev/github.com/google/uuid

80
vendor/github.com/google/uuid/dce.go generated vendored

@ -0,0 +1,80 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"encoding/binary"
"fmt"
"os"
)
// A Domain represents a Version 2 domain
type Domain byte
// Domain constants for DCE Security (Version 2) UUIDs.
const (
Person = Domain(0)
Group = Domain(1)
Org = Domain(2)
)
// NewDCESecurity returns a DCE Security (Version 2) UUID.
//
// The domain should be one of Person, Group or Org.
// On a POSIX system the id should be the users UID for the Person
// domain and the users GID for the Group. The meaning of id for
// the domain Org or on non-POSIX systems is site defined.
//
// For a given domain/id pair the same token may be returned for up to
// 7 minutes and 10 seconds.
func NewDCESecurity(domain Domain, id uint32) (UUID, error) {
uuid, err := NewUUID()
if err == nil {
uuid[6] = (uuid[6] & 0x0f) | 0x20 // Version 2
uuid[9] = byte(domain)
binary.BigEndian.PutUint32(uuid[0:], id)
}
return uuid, err
}
// NewDCEPerson returns a DCE Security (Version 2) UUID in the person
// domain with the id returned by os.Getuid.
//
// NewDCESecurity(Person, uint32(os.Getuid()))
func NewDCEPerson() (UUID, error) {
return NewDCESecurity(Person, uint32(os.Getuid()))
}
// NewDCEGroup returns a DCE Security (Version 2) UUID in the group
// domain with the id returned by os.Getgid.
//
// NewDCESecurity(Group, uint32(os.Getgid()))
func NewDCEGroup() (UUID, error) {
return NewDCESecurity(Group, uint32(os.Getgid()))
}
// Domain returns the domain for a Version 2 UUID. Domains are only defined
// for Version 2 UUIDs.
func (uuid UUID) Domain() Domain {
return Domain(uuid[9])
}
// ID returns the id for a Version 2 UUID. IDs are only defined for Version 2
// UUIDs.
func (uuid UUID) ID() uint32 {
return binary.BigEndian.Uint32(uuid[0:4])
}
func (d Domain) String() string {
switch d {
case Person:
return "Person"
case Group:
return "Group"
case Org:
return "Org"
}
return fmt.Sprintf("Domain%d", int(d))
}

12
vendor/github.com/google/uuid/doc.go generated vendored

@ -0,0 +1,12 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package uuid generates and inspects UUIDs.
//
// UUIDs are based on RFC 4122 and DCE 1.1: Authentication and Security
// Services.
//
// A UUID is a 16 byte (128 bit) array. UUIDs may be used as keys to
// maps or compared directly.
package uuid

53
vendor/github.com/google/uuid/hash.go generated vendored

@ -0,0 +1,53 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"crypto/md5"
"crypto/sha1"
"hash"
)
// Well known namespace IDs and UUIDs
var (
NameSpaceDNS = Must(Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8"))
NameSpaceURL = Must(Parse("6ba7b811-9dad-11d1-80b4-00c04fd430c8"))
NameSpaceOID = Must(Parse("6ba7b812-9dad-11d1-80b4-00c04fd430c8"))
NameSpaceX500 = Must(Parse("6ba7b814-9dad-11d1-80b4-00c04fd430c8"))
Nil UUID // empty UUID, all zeros
)
// NewHash returns a new UUID derived from the hash of space concatenated with
// data generated by h. The hash should be at least 16 byte in length. The
// first 16 bytes of the hash are used to form the UUID. The version of the
// UUID will be the lower 4 bits of version. NewHash is used to implement
// NewMD5 and NewSHA1.
func NewHash(h hash.Hash, space UUID, data []byte, version int) UUID {
h.Reset()
h.Write(space[:]) //nolint:errcheck
h.Write(data) //nolint:errcheck
s := h.Sum(nil)
var uuid UUID
copy(uuid[:], s)
uuid[6] = (uuid[6] & 0x0f) | uint8((version&0xf)<<4)
uuid[8] = (uuid[8] & 0x3f) | 0x80 // RFC 4122 variant
return uuid
}
// NewMD5 returns a new MD5 (Version 3) UUID based on the
// supplied name space and data. It is the same as calling:
//
// NewHash(md5.New(), space, data, 3)
func NewMD5(space UUID, data []byte) UUID {
return NewHash(md5.New(), space, data, 3)
}
// NewSHA1 returns a new SHA1 (Version 5) UUID based on the
// supplied name space and data. It is the same as calling:
//
// NewHash(sha1.New(), space, data, 5)
func NewSHA1(space UUID, data []byte) UUID {
return NewHash(sha1.New(), space, data, 5)
}

38
vendor/github.com/google/uuid/marshal.go generated vendored

@ -0,0 +1,38 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import "fmt"
// MarshalText implements encoding.TextMarshaler.
func (uuid UUID) MarshalText() ([]byte, error) {
var js [36]byte
encodeHex(js[:], uuid)
return js[:], nil
}
// UnmarshalText implements encoding.TextUnmarshaler.
func (uuid *UUID) UnmarshalText(data []byte) error {
id, err := ParseBytes(data)
if err != nil {
return err
}
*uuid = id
return nil
}
// MarshalBinary implements encoding.BinaryMarshaler.
func (uuid UUID) MarshalBinary() ([]byte, error) {
return uuid[:], nil
}
// UnmarshalBinary implements encoding.BinaryUnmarshaler.
func (uuid *UUID) UnmarshalBinary(data []byte) error {
if len(data) != 16 {
return fmt.Errorf("invalid UUID (got %d bytes)", len(data))
}
copy(uuid[:], data)
return nil
}

90
vendor/github.com/google/uuid/node.go generated vendored

@ -0,0 +1,90 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"sync"
)
var (
nodeMu sync.Mutex
ifname string // name of interface being used
nodeID [6]byte // hardware for version 1 UUIDs
zeroID [6]byte // nodeID with only 0's
)
// NodeInterface returns the name of the interface from which the NodeID was
// derived. The interface "user" is returned if the NodeID was set by
// SetNodeID.
func NodeInterface() string {
defer nodeMu.Unlock()
nodeMu.Lock()
return ifname
}
// SetNodeInterface selects the hardware address to be used for Version 1 UUIDs.
// If name is "" then the first usable interface found will be used or a random
// Node ID will be generated. If a named interface cannot be found then false
// is returned.
//
// SetNodeInterface never fails when name is "".
func SetNodeInterface(name string) bool {
defer nodeMu.Unlock()
nodeMu.Lock()
return setNodeInterface(name)
}
func setNodeInterface(name string) bool {
iname, addr := getHardwareInterface(name) // null implementation for js
if iname != "" && addr != nil {
ifname = iname
copy(nodeID[:], addr)
return true
}
// We found no interfaces with a valid hardware address. If name
// does not specify a specific interface generate a random Node ID
// (section 4.1.6)
if name == "" {
ifname = "random"
randomBits(nodeID[:])
return true
}
return false
}
// NodeID returns a slice of a copy of the current Node ID, setting the Node ID
// if not already set.
func NodeID() []byte {
defer nodeMu.Unlock()
nodeMu.Lock()
if nodeID == zeroID {
setNodeInterface("")
}
nid := nodeID
return nid[:]
}
// SetNodeID sets the Node ID to be used for Version 1 UUIDs. The first 6 bytes
// of id are used. If id is less than 6 bytes then false is returned and the
// Node ID is not set.
func SetNodeID(id []byte) bool {
if len(id) < 6 {
return false
}
defer nodeMu.Unlock()
nodeMu.Lock()
copy(nodeID[:], id)
ifname = "user"
return true
}
// NodeID returns the 6 byte node id encoded in uuid. It returns nil if uuid is
// not valid. The NodeID is only well defined for version 1 and 2 UUIDs.
func (uuid UUID) NodeID() []byte {
var node [6]byte
copy(node[:], uuid[10:])
return node[:]
}

12
vendor/github.com/google/uuid/node_js.go generated vendored

@ -0,0 +1,12 @@
// Copyright 2017 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build js
package uuid
// getHardwareInterface returns nil values for the JS version of the code.
// This remvoves the "net" dependency, because it is not used in the browser.
// Using the "net" library inflates the size of the transpiled JS code by 673k bytes.
func getHardwareInterface(name string) (string, []byte) { return "", nil }

33
vendor/github.com/google/uuid/node_net.go generated vendored

@ -0,0 +1,33 @@
// Copyright 2017 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !js
package uuid
import "net"
var interfaces []net.Interface // cached list of interfaces
// getHardwareInterface returns the name and hardware address of interface name.
// If name is "" then the name and hardware address of one of the system's
// interfaces is returned. If no interfaces are found (name does not exist or
// there are no interfaces) then "", nil is returned.
//
// Only addresses of at least 6 bytes are returned.
func getHardwareInterface(name string) (string, []byte) {
if interfaces == nil {
var err error
interfaces, err = net.Interfaces()
if err != nil {
return "", nil
}
}
for _, ifs := range interfaces {
if len(ifs.HardwareAddr) >= 6 && (name == "" || name == ifs.Name) {
return ifs.Name, ifs.HardwareAddr
}
}
return "", nil
}

118
vendor/github.com/google/uuid/null.go generated vendored

@ -0,0 +1,118 @@
// Copyright 2021 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"bytes"
"database/sql/driver"
"encoding/json"
"fmt"
)
var jsonNull = []byte("null")
// NullUUID represents a UUID that may be null.
// NullUUID implements the SQL driver.Scanner interface so
// it can be used as a scan destination:
//
// var u uuid.NullUUID
// err := db.QueryRow("SELECT name FROM foo WHERE id=?", id).Scan(&u)
// ...
// if u.Valid {
// // use u.UUID
// } else {
// // NULL value
// }
//
type NullUUID struct {
UUID UUID
Valid bool // Valid is true if UUID is not NULL
}
// Scan implements the SQL driver.Scanner interface.
func (nu *NullUUID) Scan(value interface{}) error {
if value == nil {
nu.UUID, nu.Valid = Nil, false
return nil
}
err := nu.UUID.Scan(value)
if err != nil {
nu.Valid = false
return err
}
nu.Valid = true
return nil
}
// Value implements the driver Valuer interface.
func (nu NullUUID) Value() (driver.Value, error) {
if !nu.Valid {
return nil, nil
}
// Delegate to UUID Value function
return nu.UUID.Value()
}
// MarshalBinary implements encoding.BinaryMarshaler.
func (nu NullUUID) MarshalBinary() ([]byte, error) {
if nu.Valid {
return nu.UUID[:], nil
}
return []byte(nil), nil
}
// UnmarshalBinary implements encoding.BinaryUnmarshaler.
func (nu *NullUUID) UnmarshalBinary(data []byte) error {
if len(data) != 16 {
return fmt.Errorf("invalid UUID (got %d bytes)", len(data))
}
copy(nu.UUID[:], data)
nu.Valid = true
return nil
}
// MarshalText implements encoding.TextMarshaler.
func (nu NullUUID) MarshalText() ([]byte, error) {
if nu.Valid {
return nu.UUID.MarshalText()
}
return jsonNull, nil
}
// UnmarshalText implements encoding.TextUnmarshaler.
func (nu *NullUUID) UnmarshalText(data []byte) error {
id, err := ParseBytes(data)
if err != nil {
nu.Valid = false
return err
}
nu.UUID = id
nu.Valid = true
return nil
}
// MarshalJSON implements json.Marshaler.
func (nu NullUUID) MarshalJSON() ([]byte, error) {
if nu.Valid {
return json.Marshal(nu.UUID)
}
return jsonNull, nil
}
// UnmarshalJSON implements json.Unmarshaler.
func (nu *NullUUID) UnmarshalJSON(data []byte) error {
if bytes.Equal(data, jsonNull) {
*nu = NullUUID{}
return nil // valid null UUID
}
err := json.Unmarshal(data, &nu.UUID)
nu.Valid = err == nil
return err
}

59
vendor/github.com/google/uuid/sql.go generated vendored

@ -0,0 +1,59 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"database/sql/driver"
"fmt"
)
// Scan implements sql.Scanner so UUIDs can be read from databases transparently.
// Currently, database types that map to string and []byte are supported. Please
// consult database-specific driver documentation for matching types.
func (uuid *UUID) Scan(src interface{}) error {
switch src := src.(type) {
case nil:
return nil
case string:
// if an empty UUID comes from a table, we return a null UUID
if src == "" {
return nil
}
// see Parse for required string format
u, err := Parse(src)
if err != nil {
return fmt.Errorf("Scan: %v", err)
}
*uuid = u
case []byte:
// if an empty UUID comes from a table, we return a null UUID
if len(src) == 0 {
return nil
}
// assumes a simple slice of bytes if 16 bytes
// otherwise attempts to parse
if len(src) != 16 {
return uuid.Scan(string(src))
}
copy((*uuid)[:], src)
default:
return fmt.Errorf("Scan: unable to scan type %T into UUID", src)
}
return nil
}
// Value implements sql.Valuer so that UUIDs can be written to databases
// transparently. Currently, UUIDs map to strings. Please consult
// database-specific driver documentation for matching types.
func (uuid UUID) Value() (driver.Value, error) {
return uuid.String(), nil
}

123
vendor/github.com/google/uuid/time.go generated vendored

@ -0,0 +1,123 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"encoding/binary"
"sync"
"time"
)
// A Time represents a time as the number of 100's of nanoseconds since 15 Oct
// 1582.
type Time int64
const (
lillian = 2299160 // Julian day of 15 Oct 1582
unix = 2440587 // Julian day of 1 Jan 1970
epoch = unix - lillian // Days between epochs
g1582 = epoch * 86400 // seconds between epochs
g1582ns100 = g1582 * 10000000 // 100s of a nanoseconds between epochs
)
var (
timeMu sync.Mutex
lasttime uint64 // last time we returned
clockSeq uint16 // clock sequence for this run
timeNow = time.Now // for testing
)
// UnixTime converts t the number of seconds and nanoseconds using the Unix
// epoch of 1 Jan 1970.
func (t Time) UnixTime() (sec, nsec int64) {
sec = int64(t - g1582ns100)
nsec = (sec % 10000000) * 100
sec /= 10000000
return sec, nsec
}
// GetTime returns the current Time (100s of nanoseconds since 15 Oct 1582) and
// clock sequence as well as adjusting the clock sequence as needed. An error
// is returned if the current time cannot be determined.
func GetTime() (Time, uint16, error) {
defer timeMu.Unlock()
timeMu.Lock()
return getTime()
}
func getTime() (Time, uint16, error) {
t := timeNow()
// If we don't have a clock sequence already, set one.
if clockSeq == 0 {
setClockSequence(-1)
}
now := uint64(t.UnixNano()/100) + g1582ns100
// If time has gone backwards with this clock sequence then we
// increment the clock sequence
if now <= lasttime {
clockSeq = ((clockSeq + 1) & 0x3fff) | 0x8000
}
lasttime = now
return Time(now), clockSeq, nil
}
// ClockSequence returns the current clock sequence, generating one if not
// already set. The clock sequence is only used for Version 1 UUIDs.
//
// The uuid package does not use global static storage for the clock sequence or
// the last time a UUID was generated. Unless SetClockSequence is used, a new
// random clock sequence is generated the first time a clock sequence is
// requested by ClockSequence, GetTime, or NewUUID. (section 4.2.1.1)
func ClockSequence() int {
defer timeMu.Unlock()
timeMu.Lock()
return clockSequence()
}
func clockSequence() int {
if clockSeq == 0 {
setClockSequence(-1)
}
return int(clockSeq & 0x3fff)
}
// SetClockSequence sets the clock sequence to the lower 14 bits of seq. Setting to
// -1 causes a new sequence to be generated.
func SetClockSequence(seq int) {
defer timeMu.Unlock()
timeMu.Lock()
setClockSequence(seq)
}
func setClockSequence(seq int) {
if seq == -1 {
var b [2]byte
randomBits(b[:]) // clock sequence
seq = int(b[0])<<8 | int(b[1])
}
oldSeq := clockSeq
clockSeq = uint16(seq&0x3fff) | 0x8000 // Set our variant
if oldSeq != clockSeq {
lasttime = 0
}
}
// Time returns the time in 100s of nanoseconds since 15 Oct 1582 encoded in
// uuid. The time is only defined for version 1 and 2 UUIDs.
func (uuid UUID) Time() Time {
time := int64(binary.BigEndian.Uint32(uuid[0:4]))
time |= int64(binary.BigEndian.Uint16(uuid[4:6])) << 32
time |= int64(binary.BigEndian.Uint16(uuid[6:8])&0xfff) << 48
return Time(time)
}
// ClockSequence returns the clock sequence encoded in uuid.
// The clock sequence is only well defined for version 1 and 2 UUIDs.
func (uuid UUID) ClockSequence() int {
return int(binary.BigEndian.Uint16(uuid[8:10])) & 0x3fff
}

43
vendor/github.com/google/uuid/util.go generated vendored

@ -0,0 +1,43 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"io"
)
// randomBits completely fills slice b with random data.
func randomBits(b []byte) {
if _, err := io.ReadFull(rander, b); err != nil {
panic(err.Error()) // rand should never fail
}
}
// xvalues returns the value of a byte as a hexadecimal digit or 255.
var xvalues = [256]byte{
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255,
255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
}
// xtob converts hex characters x1 and x2 into a byte.
func xtob(x1, x2 byte) (byte, bool) {
b1 := xvalues[x1]
b2 := xvalues[x2]
return (b1 << 4) | b2, b1 != 255 && b2 != 255
}

294
vendor/github.com/google/uuid/uuid.go generated vendored

@ -0,0 +1,294 @@
// Copyright 2018 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"bytes"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"strings"
"sync"
)
// A UUID is a 128 bit (16 byte) Universal Unique IDentifier as defined in RFC
// 4122.
type UUID [16]byte
// A Version represents a UUID's version.
type Version byte
// A Variant represents a UUID's variant.
type Variant byte
// Constants returned by Variant.
const (
Invalid = Variant(iota) // Invalid UUID
RFC4122 // The variant specified in RFC4122
Reserved // Reserved, NCS backward compatibility.
Microsoft // Reserved, Microsoft Corporation backward compatibility.
Future // Reserved for future definition.
)
const randPoolSize = 16 * 16
var (
rander = rand.Reader // random function
poolEnabled = false
poolMu sync.Mutex
poolPos = randPoolSize // protected with poolMu
pool [randPoolSize]byte // protected with poolMu
)
type invalidLengthError struct{ len int }
func (err invalidLengthError) Error() string {
return fmt.Sprintf("invalid UUID length: %d", err.len)
}
// IsInvalidLengthError is matcher function for custom error invalidLengthError
func IsInvalidLengthError(err error) bool {
_, ok := err.(invalidLengthError)
return ok
}
// Parse decodes s into a UUID or returns an error. Both the standard UUID
// forms of xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx and
// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx are decoded as well as the
// Microsoft encoding {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx} and the raw hex
// encoding: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.
func Parse(s string) (UUID, error) {
var uuid UUID
switch len(s) {
// xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
case 36:
// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
case 36 + 9:
if strings.ToLower(s[:9]) != "urn:uuid:" {
return uuid, fmt.Errorf("invalid urn prefix: %q", s[:9])
}
s = s[9:]
// {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx}
case 36 + 2:
s = s[1:]
// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
case 32:
var ok bool
for i := range uuid {
uuid[i], ok = xtob(s[i*2], s[i*2+1])
if !ok {
return uuid, errors.New("invalid UUID format")
}
}
return uuid, nil
default:
return uuid, invalidLengthError{len(s)}
}
// s is now at least 36 bytes long
// it must be of the form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
if s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' {
return uuid, errors.New("invalid UUID format")
}
for i, x := range [16]int{
0, 2, 4, 6,
9, 11,
14, 16,
19, 21,
24, 26, 28, 30, 32, 34} {
v, ok := xtob(s[x], s[x+1])
if !ok {
return uuid, errors.New("invalid UUID format")
}
uuid[i] = v
}
return uuid, nil
}
// ParseBytes is like Parse, except it parses a byte slice instead of a string.
func ParseBytes(b []byte) (UUID, error) {
var uuid UUID
switch len(b) {
case 36: // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
case 36 + 9: // urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
if !bytes.Equal(bytes.ToLower(b[:9]), []byte("urn:uuid:")) {
return uuid, fmt.Errorf("invalid urn prefix: %q", b[:9])
}
b = b[9:]
case 36 + 2: // {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx}
b = b[1:]
case 32: // xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
var ok bool
for i := 0; i < 32; i += 2 {
uuid[i/2], ok = xtob(b[i], b[i+1])
if !ok {
return uuid, errors.New("invalid UUID format")
}
}
return uuid, nil
default:
return uuid, invalidLengthError{len(b)}
}
// s is now at least 36 bytes long
// it must be of the form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
if b[8] != '-' || b[13] != '-' || b[18] != '-' || b[23] != '-' {
return uuid, errors.New("invalid UUID format")
}
for i, x := range [16]int{
0, 2, 4, 6,
9, 11,
14, 16,
19, 21,
24, 26, 28, 30, 32, 34} {
v, ok := xtob(b[x], b[x+1])
if !ok {
return uuid, errors.New("invalid UUID format")
}
uuid[i] = v
}
return uuid, nil
}
// MustParse is like Parse but panics if the string cannot be parsed.
// It simplifies safe initialization of global variables holding compiled UUIDs.
func MustParse(s string) UUID {
uuid, err := Parse(s)
if err != nil {
panic(`uuid: Parse(` + s + `): ` + err.Error())
}
return uuid
}
// FromBytes creates a new UUID from a byte slice. Returns an error if the slice
// does not have a length of 16. The bytes are copied from the slice.
func FromBytes(b []byte) (uuid UUID, err error) {
err = uuid.UnmarshalBinary(b)
return uuid, err
}
// Must returns uuid if err is nil and panics otherwise.
func Must(uuid UUID, err error) UUID {
if err != nil {
panic(err)
}
return uuid
}
// String returns the string form of uuid, xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
// , or "" if uuid is invalid.
func (uuid UUID) String() string {
var buf [36]byte
encodeHex(buf[:], uuid)
return string(buf[:])
}
// URN returns the RFC 2141 URN form of uuid,
// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx, or "" if uuid is invalid.
func (uuid UUID) URN() string {
var buf [36 + 9]byte
copy(buf[:], "urn:uuid:")
encodeHex(buf[9:], uuid)
return string(buf[:])
}
func encodeHex(dst []byte, uuid UUID) {
hex.Encode(dst, uuid[:4])
dst[8] = '-'
hex.Encode(dst[9:13], uuid[4:6])
dst[13] = '-'
hex.Encode(dst[14:18], uuid[6:8])
dst[18] = '-'
hex.Encode(dst[19:23], uuid[8:10])
dst[23] = '-'
hex.Encode(dst[24:], uuid[10:])
}
// Variant returns the variant encoded in uuid.
func (uuid UUID) Variant() Variant {
switch {
case (uuid[8] & 0xc0) == 0x80:
return RFC4122
case (uuid[8] & 0xe0) == 0xc0:
return Microsoft
case (uuid[8] & 0xe0) == 0xe0:
return Future
default:
return Reserved
}
}
// Version returns the version of uuid.
func (uuid UUID) Version() Version {
return Version(uuid[6] >> 4)
}
func (v Version) String() string {
if v > 15 {
return fmt.Sprintf("BAD_VERSION_%d", v)
}
return fmt.Sprintf("VERSION_%d", v)
}
func (v Variant) String() string {
switch v {
case RFC4122:
return "RFC4122"
case Reserved:
return "Reserved"
case Microsoft:
return "Microsoft"
case Future:
return "Future"
case Invalid:
return "Invalid"
}
return fmt.Sprintf("BadVariant%d", int(v))
}
// SetRand sets the random number generator to r, which implements io.Reader.
// If r.Read returns an error when the package requests random data then
// a panic will be issued.
//
// Calling SetRand with nil sets the random number generator to the default
// generator.
func SetRand(r io.Reader) {
if r == nil {
rander = rand.Reader
return
}
rander = r
}
// EnableRandPool enables internal randomness pool used for Random
// (Version 4) UUID generation. The pool contains random bytes read from
// the random number generator on demand in batches. Enabling the pool
// may improve the UUID generation throughput significantly.
//
// Since the pool is stored on the Go heap, this feature may be a bad fit
// for security sensitive applications.
//
// Both EnableRandPool and DisableRandPool are not thread-safe and should
// only be called when there is no possibility that New or any other
// UUID Version 4 generation function will be called concurrently.
func EnableRandPool() {
poolEnabled = true
}
// DisableRandPool disables the randomness pool if it was previously
// enabled with EnableRandPool.
//
// Both EnableRandPool and DisableRandPool are not thread-safe and should
// only be called when there is no possibility that New or any other
// UUID Version 4 generation function will be called concurrently.
func DisableRandPool() {
poolEnabled = false
defer poolMu.Unlock()
poolMu.Lock()
poolPos = randPoolSize
}

44
vendor/github.com/google/uuid/version1.go generated vendored

@ -0,0 +1,44 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"encoding/binary"
)
// NewUUID returns a Version 1 UUID based on the current NodeID and clock
// sequence, and the current time. If the NodeID has not been set by SetNodeID
// or SetNodeInterface then it will be set automatically. If the NodeID cannot
// be set NewUUID returns nil. If clock sequence has not been set by
// SetClockSequence then it will be set automatically. If GetTime fails to
// return the current NewUUID returns nil and an error.
//
// In most cases, New should be used.
func NewUUID() (UUID, error) {
var uuid UUID
now, seq, err := GetTime()
if err != nil {
return uuid, err
}
timeLow := uint32(now & 0xffffffff)
timeMid := uint16((now >> 32) & 0xffff)
timeHi := uint16((now >> 48) & 0x0fff)
timeHi |= 0x1000 // Version 1
binary.BigEndian.PutUint32(uuid[0:], timeLow)
binary.BigEndian.PutUint16(uuid[4:], timeMid)
binary.BigEndian.PutUint16(uuid[6:], timeHi)
binary.BigEndian.PutUint16(uuid[8:], seq)
nodeMu.Lock()
if nodeID == zeroID {
setNodeInterface("")
}
copy(uuid[10:], nodeID[:])
nodeMu.Unlock()
return uuid, nil
}

76
vendor/github.com/google/uuid/version4.go generated vendored

@ -0,0 +1,76 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import "io"
// New creates a new random UUID or panics. New is equivalent to
// the expression
//
// uuid.Must(uuid.NewRandom())
func New() UUID {
return Must(NewRandom())
}
// NewString creates a new random UUID and returns it as a string or panics.
// NewString is equivalent to the expression
//
// uuid.New().String()
func NewString() string {
return Must(NewRandom()).String()
}
// NewRandom returns a Random (Version 4) UUID.
//
// The strength of the UUIDs is based on the strength of the crypto/rand
// package.
//
// Uses the randomness pool if it was enabled with EnableRandPool.
//
// A note about uniqueness derived from the UUID Wikipedia entry:
//
// Randomly generated UUIDs have 122 random bits. One's annual risk of being
// hit by a meteorite is estimated to be one chance in 17 billion, that
// means the probability is about 0.00000000006 (6 × 10−11),
// equivalent to the odds of creating a few tens of trillions of UUIDs in a
// year and having one duplicate.
func NewRandom() (UUID, error) {
if !poolEnabled {
return NewRandomFromReader(rander)
}
return newRandomFromPool()
}
// NewRandomFromReader returns a UUID based on bytes read from a given io.Reader.
func NewRandomFromReader(r io.Reader) (UUID, error) {
var uuid UUID
_, err := io.ReadFull(r, uuid[:])
if err != nil {
return Nil, err
}
uuid[6] = (uuid[6] & 0x0f) | 0x40 // Version 4
uuid[8] = (uuid[8] & 0x3f) | 0x80 // Variant is 10
return uuid, nil
}
func newRandomFromPool() (UUID, error) {
var uuid UUID
poolMu.Lock()
if poolPos == randPoolSize {
_, err := io.ReadFull(rander, pool[:])
if err != nil {
poolMu.Unlock()
return Nil, err
}
poolPos = 0
}
copy(uuid[:], pool[poolPos:(poolPos+16)])
poolPos += 16
poolMu.Unlock()
uuid[6] = (uuid[6] & 0x0f) | 0x40 // Version 4
uuid[8] = (uuid[8] & 0x3f) | 0x80 // Variant is 10
return uuid, nil
}

2
vendor/github.com/klauspost/compress/.gitattributes generated vendored

@ -0,0 +1,2 @@
* -text
*.bin -text -diff

25
vendor/github.com/klauspost/compress/.gitignore generated vendored

@ -0,0 +1,25 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
/s2/cmd/_s2sx/sfx-exe

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save