00001 """\ 00002 @file tokenstream.py 00003 @brief Message template parsing utility class 00004 00005 $LicenseInfo:firstyear=2007&license=mit$ 00006 00007 Copyright (c) 2007-2008, Linden Research, Inc. 00008 00009 Permission is hereby granted, free of charge, to any person obtaining a copy 00010 of this software and associated documentation files (the "Software"), to deal 00011 in the Software without restriction, including without limitation the rights 00012 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 00013 copies of the Software, and to permit persons to whom the Software is 00014 furnished to do so, subject to the following conditions: 00015 00016 The above copyright notice and this permission notice shall be included in 00017 all copies or substantial portions of the Software. 00018 00019 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 00020 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 00021 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 00022 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 00023 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 00024 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 00025 THE SOFTWARE. 00026 $/LicenseInfo$ 00027 """ 00028 00029 import re 00030 00031 class _EOF(object): 00032 pass 00033 00034 EOF = _EOF() 00035 00036 class _LineMarker(int): 00037 pass 00038 00039 _commentRE = re.compile(r'//.*') 00040 _symbolRE = re.compile(r'[a-zA-Z_][a-zA-Z_0-9]*') 00041 _integerRE = re.compile(r'(0x[0-9A-Fa-f]+|0\d*|[1-9]\d*)') 00042 _floatRE = re.compile(r'\d+(\.\d*)?') 00043 00044 00045 class ParseError(Exception): 00046 def __init__(self, stream, reason): 00047 self.line = stream.line 00048 self.context = stream._context() 00049 self.reason = reason 00050 00051 def _contextString(self): 00052 c = [ ] 00053 for t in self.context: 00054 if isinstance(t, _LineMarker): 00055 break 00056 c.append(t) 00057 return " ".join(c) 00058 00059 def __str__(self): 00060 return "line %d: %s @ ... %s" % ( 00061 self.line, self.reason, self._contextString()) 00062 00063 def __nonzero__(self): 00064 return False 00065 00066 00067 def _optionText(options): 00068 n = len(options) 00069 if n == 1: 00070 return '"%s"' % options[0] 00071 return '"' + '", "'.join(options[0:(n-1)]) + '" or "' + options[-1] + '"' 00072 00073 00074 class TokenStream(object): 00075 def __init__(self): 00076 self.line = 0 00077 self.tokens = [ ] 00078 00079 def fromString(self, string): 00080 return self.fromLines(string.split('\n')) 00081 00082 def fromFile(self, file): 00083 return self.fromLines(file) 00084 00085 def fromLines(self, lines): 00086 i = 0 00087 for line in lines: 00088 i += 1 00089 self.tokens.append(_LineMarker(i)) 00090 self.tokens.extend(_commentRE.sub(" ", line).split()) 00091 self._consumeLines() 00092 return self 00093 00094 def consume(self): 00095 if not self.tokens: 00096 return EOF 00097 t = self.tokens.pop(0) 00098 self._consumeLines() 00099 return t 00100 00101 def _consumeLines(self): 00102 while self.tokens and isinstance(self.tokens[0], _LineMarker): 00103 self.line = self.tokens.pop(0) 00104 00105 def peek(self): 00106 if not self.tokens: 00107 return EOF 00108 return self.tokens[0] 00109 00110 def want(self, t): 00111 if t == self.peek(): 00112 return self.consume() 00113 return ParseError(self, 'expected "%s"' % t) 00114 00115 def wantOneOf(self, options): 00116 assert len(options) 00117 if self.peek() in options: 00118 return self.consume() 00119 return ParseError(self, 'expected one of %s' % _optionText(options)) 00120 00121 def wantEOF(self): 00122 return self.want(EOF) 00123 00124 def wantRE(self, re, message=None): 00125 t = self.peek() 00126 if t != EOF: 00127 m = re.match(t) 00128 if m and m.end() == len(t): 00129 return self.consume() 00130 if not message: 00131 message = "expected match for r'%s'" % re.pattern 00132 return ParseError(self, message) 00133 00134 def wantSymbol(self): 00135 return self.wantRE(_symbolRE, "expected symbol") 00136 00137 def wantInteger(self): 00138 return self.wantRE(_integerRE, "expected integer") 00139 00140 def wantFloat(self): 00141 return self.wantRE(_floatRE, "expected float") 00142 00143 def _context(self): 00144 n = min(5, len(self.tokens)) 00145 return self.tokens[0:n] 00146 00147 def require(self, t): 00148 if t: 00149 return t 00150 if isinstance(t, ParseError): 00151 raise t 00152 else: 00153 raise ParseError(self, "unmet requirement") 00154