anko源代码阅读之lex文件阅读(三)

2018/03/23

这篇文章是在阅读GitHub - mattn/anko: Scriptable interpreter written in golang时候的笔记

diff --git parser/lexer.go parser/lexer.go
 // Package parser implements parser for anko.
 package parser

 import (
 	"errors"
 	"fmt"
 	"unicode"

 	"github.com/mattn/anko/ast"
 )

 const (
 	// EOF is short for End of file.
+	// 文件的末尾
 	EOF = -1
+	// 行的末尾
 	// EOL is short for End of line.
 	EOL = '\n'
 )

 // Error provides a convenient interface for handling runtime error.
 // It can be Error interface with type cast which can call Pos().
 type Error struct {
 	Message  string
 	Pos      ast.Position
 	Filename string
 	Fatal    bool
 }

 // Error returns the error message.
 func (e *Error) Error() string {
 	return e.Message
 }

 // Scanner stores informations for lexer.
 type Scanner struct {
-	src      []rune
+	src []rune
+	// 当前scan的位置
 	offset   int
 	lineHead int
 	line     int
 }

 // opName is correction of operation names.
+// 这里记录着关键字
 var opName = map[string]int{
 	"func":     FUNC,
 	"return":   RETURN,
 	"var":      VAR,
 	"throw":    THROW,
 	"if":       IF,
 	"for":      FOR,
 	"break":    BREAK,
 	"continue": CONTINUE,
 	"in":       IN,
 	"else":     ELSE,
 	"new":      NEW,
 	"true":     TRUE,
 	"false":    FALSE,
 	"nil":      NIL,
 	"module":   MODULE,
 	"try":      TRY,
 	"catch":    CATCH,
 	"finally":  FINALLY,
 	"switch":   SWITCH,
 	"case":     CASE,
 	"default":  DEFAULT,
 	"go":       GO,
 	"chan":     CHAN,
 	"make":     MAKE,
 	"type":     TYPE,
 	"len":      LEN,
 	"delete":   DELETE,
 }

 // Init resets code to scan.
+// Init string -> []rune
 func (s *Scanner) Init(src string) {
 	s.src = []rune(src)
 }

 // Scan analyses token, and decide identify or literals.
+// scan是扫描的主逻辑
 func (s *Scanner) Scan() (tok int, lit string, pos ast.Position, err error) {
 retry:
-	s.skipBlank()
+	s.skipBlank() // 跳过空白
 	pos = s.pos()
 	switch ch := s.peek(); {
 	case isLetter(ch):
+		// 如果是字母,获取 scanIdentifier
 		lit, err = s.scanIdentifier()
 		if err != nil {
 			return
 		}
+
+		// 是否是已经记录的关键字
 		if name, ok := opName[lit]; ok {
 			tok = name
 		} else {
+			// IDENT
 			tok = IDENT
 		}
 	case isDigit(ch):
+		// 如果是数字,获取数字,NUMBER
 		tok = NUMBER
 		lit, err = s.scanNumber()
 		if err != nil {
 			return
 		}
 	case ch == '"':
+		// 如果是 ",扫描下一个",并设置为string
 		tok = STRING
 		lit, err = s.scanString('"')
 		if err != nil {
 			return
 		}
 	case ch == '\'':
+		// 如果是 \',扫描下一个\',并设置为string
 		tok = STRING
 		lit, err = s.scanString('\'')
 		if err != nil {
 			return
 		}
 	case ch == '`':
+		// 如果是 `,扫描下一个`,并设置为string
 		tok = STRING
 		lit, err = s.scanRawString()
 		if err != nil {
 			return
 		}
 	default:
+		// 剩下这些有一个共同点,需要 s.next()
 		switch ch {
 		case EOF:
+			// 文件末尾
 			tok = EOF
 		case '#':
+			// 注释,把这一行葬送掉,然后retry
 			for !isEOL(s.peek()) {
 				s.next()
 			}
 			goto retry
 		case '!':
 			s.next()
 			switch s.peek() {
 			case '=':
+				// `!=`
 				tok = NEQ
 				lit = "!="
 			default:
+				// `!`
 				s.back()
 				tok = int(ch)
 				lit = string(ch)
 			}
 		case '=':
 			s.next()
 			switch s.peek() {
 			case '=':
+				// `==`
 				tok = EQEQ
 				lit = "=="
 			default:
+				// `=`
 				s.back()
 				tok = int(ch)
 				lit = string(ch)
 			}
 		case '+':
 			s.next()
 			switch s.peek() {
 			case '+':
+				// `++`
 				tok = PLUSPLUS
 				lit = "++"
 			case '=':
+				// `+=`
 				tok = PLUSEQ
 				lit = "+="
 			default:
+				// `+`
 				s.back()
 				tok = int(ch)
 				lit = string(ch)
 			}
 		case '-':
 			s.next()
 			switch s.peek() {
 			case '-':
+				// `--`
 				tok = MINUSMINUS
 				lit = "--"
 			case '=':
+				// `-=`
 				tok = MINUSEQ
 				lit = "-="
 			default:
+				// `-`
 				s.back()
 				tok = int(ch)
 				lit = string(ch)
 			}
 		case '*':
 			s.next()
 			switch s.peek() {
 			case '*':
+				// `**`
 				tok = POW
 				lit = "**"
 			case '=':
+				// `*=`
 				tok = MULEQ
 				lit = "*="
 			default:
+				// `*`
 				s.back()
 				tok = int(ch)
 				lit = string(ch)
 			}
 		case '/':
 			s.next()
 			switch s.peek() {
 			case '=':
+				// `/=`
 				tok = DIVEQ
 				lit = "/="
 			default:
+				// `/`
 				s.back()
 				tok = int(ch)
 				lit = string(ch)
 			}
 		case '>':
 			s.next()
 			switch s.peek() {
 			case '=':
+				// `>=`
 				tok = GE
 				lit = ">="
 			case '>':
+				// `>>`
 				tok = SHIFTRIGHT
 				lit = ">>"
 			default:
+				// `>`
 				s.back()
 				tok = int(ch)
 				lit = string(ch)
 			}
 		case '<':
 			s.next()
 			switch s.peek() {
 			case '-':
+				// `<-`
 				tok = OPCHAN
 				lit = "<-"
 			case '=':
+				// `<=`
 				tok = LE
 				lit = "<="
 			case '<':
+				// `<<`
 				tok = SHIFTLEFT
 				lit = "<<"
 			default:
+				// `<`
 				s.back()
 				tok = int(ch)
 				lit = string(ch)
 			}
 		case '|':
 			s.next()
 			switch s.peek() {
 			case '|':
+				// `||`
 				tok = OROR
 				lit = "||"
 			case '=':
+				// `|=`
 				tok = OREQ
 				lit = "|="
 			default:
+				// `|`
 				s.back()
 				tok = int(ch)
 				lit = string(ch)
 			}
 		case '&':
 			s.next()
 			switch s.peek() {
 			case '&':
+				// `&&`
 				tok = ANDAND
 				lit = "&&"
 			case '=':
+				// `&=`
 				tok = ANDEQ
 				lit = "&="
 			default:
+				// `&`
 				s.back()
 				tok = int(ch)
 				lit = string(ch)
 			}
 		case '.':
 			s.next()
 			if s.peek() == '.' {
 				s.next()
 				if s.peek() == '.' {
+					// `...`
 					tok = VARARG
 				} else {
+					// `..` error
 					err = fmt.Errorf("syntax error on '%v' at %v:%v", string(ch), pos.Line, pos.Column)
 					return
 				}
 			} else {
+				// `.`
 				s.back()
 				tok = int(ch)
 				lit = string(ch)
 			}
 		case '\n', '(', ')', ':', ';', '%', '?', '{', '}', '[', ']', ',', '^':
+			// 这些不能后面接个字符,构成新的字符,所以直接返回 `int(ch)`
 			tok = int(ch)
 			lit = string(ch)
 		default:
+			// error
 			err = fmt.Errorf("syntax error on '%v' at %v:%v", string(ch), pos.Line, pos.Column)
 			tok = int(ch)
 			lit = string(ch)
 			return
 		}
 		s.next()
 	}
 	return
 }

 // isLetter returns true if the rune is a letter for identity.
+// 是否是字母:unicode.IsLetter || '_'
 func isLetter(ch rune) bool {
 	return unicode.IsLetter(ch) || ch == '_'
 }

 // isDigit returns true if the rune is a number.
+// 是否是数字的字符集:'0'~'9'
 func isDigit(ch rune) bool {
 	return '0' <= ch && ch <= '9'
 }

 // isHex returns true if the rune is a hex digits.
+// 返回是否是16进制的字符:'0'~'9' || 'a'~'f' || 'A'~'F'
 func isHex(ch rune) bool {
 	return ('0' <= ch && ch <= '9') || ('a' <= ch && ch <= 'f') || ('A' <= ch && ch <= 'F')
 }

 // isEOL returns true if the rune is at end-of-line or end-of-file.
+// 是否是文件末尾或者行的末尾: '\n' ch == -1
 func isEOL(ch rune) bool {
 	return ch == '\n' || ch == -1
 }

 // isBlank returns true if the rune is empty character..
+// 是否为空:' ', '\t', '\r'
 func isBlank(ch rune) bool {
 	return ch == ' ' || ch == '\t' || ch == '\r'
 }

 // peek returns current rune in the code.
+// 如果是文件末尾,那么返回EOF,否则返回当前位置的字符
 func (s *Scanner) peek() rune {
 	if s.reachEOF() {
 		return EOF
 	}
 	return s.src[s.offset]
 }

 // next moves offset to next.
+// 如果没有到达文件末尾,那么:
+//     游标加1
+//     且:如果当前位置是\n,那么
+//         s.lineHead = s.offset + 1
+//         s.line++
 func (s *Scanner) next() {
 	if !s.reachEOF() {
 		if s.peek() == '\n' {
 			s.lineHead = s.offset + 1
 			s.line++
 		}
 		s.offset++
 	}
 }

 // current returns the current offset.
+// 获取当前的游标
 func (s *Scanner) current() int {
 	return s.offset
 }

 // offset sets the offset value.
+// 设置当前游标
 func (s *Scanner) set(o int) {
 	s.offset = o
 }

 // back moves back offset once to top.
+// 游标后移一位
 func (s *Scanner) back() {
 	s.offset--
 }

 // reachEOF returns true if offset is at end-of-file.
+// 是否达到文件末尾:offset >= len
 func (s *Scanner) reachEOF() bool {
 	return len(s.src) <= s.offset
 }

 // pos returns the position of current.
+// 返回当前位置的 ast.Position,
+// Line:s.line + 1
+// Column: s.offset - s.lineHead + 1
 func (s *Scanner) pos() ast.Position {
 	return ast.Position{Line: s.line + 1, Column: s.offset - s.lineHead + 1}
 }

 // skipBlank moves position into non-black character.
+// 跳过blank:空字符,\t,\n
 func (s *Scanner) skipBlank() {
 	for isBlank(s.peek()) {
 		s.next()
 	}
 }

 // scanIdentifier returns identifier beginning at current position.
+// 返回从当前位置开始的,字母和数字的最长串
 func (s *Scanner) scanIdentifier() (string, error) {
 	var ret []rune
 	for {
 		if !isLetter(s.peek()) && !isDigit(s.peek()) {
 			break
 		}
 		ret = append(ret, s.peek())
 		s.next()
 	}
 	return string(ret), nil
 }

 // scanNumber returns number beginning at current position.
+// 返回当前位置开始的数字
 func (s *Scanner) scanNumber() (string, error) {
 	var ret []rune
 	ch := s.peek()
 	ret = append(ret, ch)
 	s.next()
 	if ch == '0' && s.peek() == 'x' {
 		ret = append(ret, s.peek())
 		s.next()
 		for isHex(s.peek()) {
 			ret = append(ret, s.peek())
 			s.next()
 		}
 	} else {
 		for isDigit(s.peek()) || s.peek() == '.' {
 			ret = append(ret, s.peek())
 			s.next()
 		}
 		if s.peek() == 'e' {
 			ret = append(ret, s.peek())
 			s.next()
 			if isDigit(s.peek()) || s.peek() == '+' || s.peek() == '-' {
 				ret = append(ret, s.peek())
 				s.next()
 				for isDigit(s.peek()) || s.peek() == '.' {
 					ret = append(ret, s.peek())
 					s.next()
 				}
 			}
 			for isDigit(s.peek()) || s.peek() == '.' {
 				ret = append(ret, s.peek())
 				s.next()
 			}
 		}
 		if isLetter(s.peek()) {
 			return "", errors.New("identifier starts immediately after numeric literal")
 		}
 	}
 	return string(ret), nil
 }

 // scanRawString returns raw-string starting at current position.
+// row-string,返回 当前位置开始的,` 之前的
 func (s *Scanner) scanRawString() (string, error) {
 	var ret []rune
 	for {
 		s.next()
 		if s.peek() == EOF {
 			return "", errors.New("unexpected EOF")
 		}
 		if s.peek() == '`' {
 			s.next()
 			break
 		}
 		ret = append(ret, s.peek())
 	}
 	return string(ret), nil
 }

 // scanString returns string starting at current position.
 // This handles backslash escaping.
+// 扫描,直到遇到某一个字符,比如scanString('"'),就会返回当前位置开始的,"之前的字符串
 func (s *Scanner) scanString(l rune) (string, error) {
 	var ret []rune
 eos:
 	for {
 		s.next()
 		switch s.peek() {
 		case EOL:
 			return "", errors.New("unexpected EOL")
 		case EOF:
 			return "", errors.New("unexpected EOF")
 		case l:
 			s.next()
 			break eos
 		case '\\':
 			s.next()
 			switch s.peek() {
 			case 'b':
 				ret = append(ret, '\b')
 				continue
 			case 'f':
 				ret = append(ret, '\f')
 				continue
 			case 'r':
 				ret = append(ret, '\r')
 				continue
 			case 'n':
 				ret = append(ret, '\n')
 				continue
 			case 't':
 				ret = append(ret, '\t')
 				continue
 			}
 			ret = append(ret, s.peek())
 			continue
 		default:
 			ret = append(ret, s.peek())
 		}
 	}
 	return string(ret), nil
 }

 // Lexer provides interface to parse codes.
 type Lexer struct {
 	s     *Scanner
 	lit   string
 	pos   ast.Position
 	e     error
 	stmts []ast.Stmt
 }

 // Lex scans the token and literals.
+// 这个实现了xx接口,会被用来解析各个部分
 func (l *Lexer) Lex(lval *yySymType) int {
+	// 扫描一次
 	tok, lit, pos, err := l.s.Scan()
 	if err != nil {
 		l.e = &Error{Message: err.Error(), Pos: pos, Fatal: true}
 	}
 	lval.tok = ast.Token{Tok: tok, Lit: lit}
 	lval.tok.SetPosition(pos)
 	l.lit = lit
 	l.pos = pos
 	return tok
 }

 // Error sets parse error.
 func (l *Lexer) Error(msg string) {
 	l.e = &Error{Message: msg, Pos: l.pos, Fatal: false}
 }

 // Parse provides way to parse the code using Scanner.
 func Parse(s *Scanner) ([]ast.Stmt, error) {
 	l := Lexer{s: s}
 	if yyParse(&l) != 0 {
 		return nil, l.e
 	}
+
 	return l.stmts, l.e
 }

 // EnableErrorVerbose enabled verbose errors from the parser
+// 打开详细错误信息
 func EnableErrorVerbose() {
 	yyErrorVerbose = true
 }

 // ParseSrc provides way to parse the code from source.
+// 解析的入口
 func ParseSrc(src string) ([]ast.Stmt, error) {
+	// new一个Scanner
 	scanner := &Scanner{
 		src: []rune(src),
 	}
-	return Parse(scanner)
+	b, err := Parse(scanner)
+	return b, err
 }