Tokenization.swift 6.82 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
// Copyright (c) 2016 Anarchy Tools Contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

15
import atfoundation
16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45

public enum TokenType {
    case Identifier
    case OpenParen
    case CloseParen
    case OpenBracket
    case CloseBracket
    case OpenBrace
    case CloseBrace
    case StringLiteral
    case Terminal
    case Colon
    case Comment

    case Unknown
    case EOF
}

public func ==(lhs: Token, rhs: Token) -> Bool {
    return lhs.type == rhs.type &&
           lhs.line == rhs.line &&
           lhs.column == rhs.column &&
           lhs.value == rhs.value
}

public struct Token: Equatable {
    public let value: String
    public let line: Int
    public let column: Int
    public let type: TokenType
46

47 48 49 50 51 52 53 54 55
    public init(type: TokenType, value: String = "", line: Int = 0, column: Int = 0) {
        self.type = type
        self.value = value
        self.line = line
        self.column = column
    }
}

func isValidIdentifierSignalCharacter(c: Character?) -> Bool {
56 57 58 59
    guard let c = c else {
        return false
    }
    return Charset.isLetter(character: c)
60 61 62
}

func isValidIdenitifierCharacter(c: Character?) -> Bool {
63 64 65 66
    guard let c = c else {
        return false
    }
    return Charset.isLetter(character: c) || c == "-" || c == "." || c == "/"
67 68 69
}

func isWhitespace(c: Character?) -> Bool {
70 71 72 73
    guard let c = c else {
        return false
    }
    return Charset.isWhitespace(character: c)
74 75
}

76
final public class Lexer {
77 78
    var scanner: Scanner
    var current: Token? = nil
79

80
    var shouldStall = false
81

82 83 84 85 86 87 88 89 90
    public init(scanner: Scanner) {
        self.scanner = scanner
    }

    public func next() -> Token? {
        if shouldStall {
            shouldStall = false
            return current
        }
91

92 93 94 95 96
        func work() -> Token {
            if scanner.next() == nil { return Token(type: .EOF) }

            scanner.stall()

97
            while let info = scanner.next() where isWhitespace(c: info.character) {}
98 99 100 101 102 103 104
            scanner.stall()

            guard let next = scanner.next() else { return Token(type: .EOF) }

            if next.character == "\n" {
                return Token(type: .Terminal, value: "\n", line: next.line, column: next.column)
            }
105
            else if isValidIdentifierSignalCharacter(c: next.character) {
106
                var content = String(next.character!)
107
                while let info = scanner.next() where isValidIdenitifierCharacter(c: info.character) {
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141
                    content.append(info.character!)
                }
                scanner.stall()

                return Token(type: .Identifier, value: content, line: next.line, column: next.column)
            }
            else if next.character == "(" {
                return Token(type: .OpenParen, value: "(", line: next.line, column: next.column)
            }
            else if next.character == ")" {
                return Token(type: .CloseParen, value: ")", line: next.line, column: next.column)
            }
            else if next.character == "[" {
                return Token(type: .OpenBracket, value: "[", line: next.line, column: next.column)
            }
            else if next.character == "]" {
                return Token(type: .CloseBracket, value: "]", line: next.line, column: next.column)
            }
            else if next.character == "{" {
                return Token(type: .OpenBrace, value: "{", line: next.line, column: next.column)
            }
            else if next.character == "}" {
                return Token(type: .CloseBrace, value: "}", line: next.line, column: next.column)
            }
            else if next.character == ":" {
                return Token(type: .Colon, value: ":", line: next.line, column: next.column)
            }
            else if next.character == ";" {
                let column = scanner.peek()!.column
                let line = scanner.peek()!.line
                var comment = ""

                while let info = scanner.next() where info.character == ";" {}
                scanner.stall()
142

143 144 145 146 147 148 149
                while let info = scanner.next() where info.character != "\n" {
                    comment.append(info.character!)
                }

                return Token(type: .Comment, value: comment, line: line, column: column)
            }
            else if next.character == "\"" {
150
                var content = ""
151
                while let info = scanner.next() where info.character != "\"" {
152 153 154 155 156 157 158 159 160 161 162 163 164
                    if info.character == "\\" {
                        let escaped = scanner.next()
                        let char = escaped?.character
                        
                        switch escaped?.character {
                        case _ where char == "t": content.append("\t"); break
                        case _ where char == "b": fatalError("Unsupported escape sequence: \\b")
                        case _ where char == "n": content.append("\n"); break
                        case _ where char == "r": content.append("\r"); break
                        case _ where char == "f": fatalError("Unsupported escape sequence: \\f")
                        case _ where char == "'": content.append("'"); break
                        case _ where char == "\"": content.append("\""); break
                        case _ where char == "\\": content.append("\\"); break
165
                        case _ where char == "$": content.append("\\$"); break
166 167 168 169 170 171 172
                        default:
                            fatalError("Unsupposed escape sequence: \\\(escaped?.character)")
                        }
                    }
                    else {
                        content.append(info.character!)
                    }
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
                }

                return Token(type: .StringLiteral, value: content, line: next.line, column: next.column)
            }
            else {
                return Token(type: .Unknown, value: String(next.character!), line: next.line, column: next.column)
            }
        }

        if self.current?.type == .EOF {
            self.current = nil
        }
        else {
            self.current = work()
        }
188

189 190 191 192 193 194 195 196 197 198 199 200 201 202
        return self.current
    }

    func tokenize() -> [Token] {
        var tokens = [Token]()

        while let token = self.next() { tokens.append(token) }

        return tokens
    }

    public func peek() -> Token? {
        return current
    }
203

204 205 206 207
    public func stall() {
        shouldStall = true
    }
}