You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
23 lines
372 B
23 lines
372 B
package lexer |
|
|
|
import ( |
|
"io" |
|
|
|
"github.com/goccy/go-yaml/scanner" |
|
"github.com/goccy/go-yaml/token" |
|
) |
|
|
|
// Tokenize split to token instances from string |
|
func Tokenize(src string) token.Tokens { |
|
var s scanner.Scanner |
|
s.Init(src) |
|
var tokens token.Tokens |
|
for { |
|
subTokens, err := s.Scan() |
|
if err == io.EOF { |
|
break |
|
} |
|
tokens.Add(subTokens...) |
|
} |
|
return tokens |
|
}
|
|
|