001

.gitignore

@@ -0,0 +1 @@
+.idea

go.mod

@@ -0,0 +1,3 @@
+module lexer-and-parser-thing
+
+go 1.24.0

main.go

@@ -0,0 +1,16 @@
+package main
+
+import "fmt"
+
+const fynCode = `12 + 34 * 56 / 78 - 90`
+
+func main() {
+	scanner := Scanner{source: fynCode}
+
+	token := scanner.Consume()
+	for token.kind != "eof" {
+		fmt.Println(token)
+
+		token = scanner.Consume()
+	}
+}

scanner.go

@@ -0,0 +1,74 @@
+package main
+
+type Scanner struct {
+	source string
+	cursor int
+}
+
+func (s *Scanner) Consume() Token {
+	if s.cursor >= len(s.source) {
+		return Token{kind: "eof", lexeme: "EOF"}
+	}
+
+	s.readWhile(isWhitespace)
+
+	next := s.peek()
+
+	if isDigit(next) {
+		lexeme := s.readWhile(isDigit)
+
+		return Token{kind: "number", lexeme: lexeme}
+	}
+
+	kind, ok := kinds[next]
+	if ok {
+		lexeme := s.read()
+
+		return Token{kind: kind, lexeme: string(lexeme)}
+	}
+
+	lexeme := s.readWhile(notWhitespace)
+
+	return Token{kind: "unknown", lexeme: lexeme}
+}
+
+func (s *Scanner) peek() rune {
+	if s.cursor >= len(s.source) {
+		return 0
+	}
+
+	return rune(s.source[s.cursor])
+}
+
+func (s *Scanner) read() rune {
+	c := rune(s.source[s.cursor])
+
+	s.cursor += 1
+
+	return c
+}
+
+func (s *Scanner) readWhile(predicate func(rune) bool) string {
+	lexeme := ""
+
+	c := s.peek()
+	for s.cursor < len(s.source) && predicate(c) {
+		lexeme += string(s.read())
+
+		c = s.peek()
+	}
+
+	return lexeme
+}
+
+func isWhitespace(c rune) bool {
+	return c == ' ' || c == '\t' || c == '\n'
+}
+
+func notWhitespace(c rune) bool {
+	return !isWhitespace(c)
+}
+
+func isDigit(c rune) bool {
+	return '0' <= c && c <= '9'
+}

token.go

@@ -0,0 +1,13 @@
+package main
+
+var kinds = map[rune]string{
+	'+': "plus",
+	'-': "minus",
+	'*': "asterisk",
+	'/': "slash",
+}
+
+type Token struct {
+	kind   string
+	lexeme string
+}