37 lines
1.4 KiB
Python
37 lines
1.4 KiB
Python
import re
|
||
from collections import namedtuple
|
||
|
||
if __name__ == "__main__":
|
||
text = 'foo = 23 + 42 * 10'
|
||
# 我们想要将这个字符串解析成这个样子：
|
||
# tokens = [('NAME', 'foo'), ('EQ', '='), ('NUMS', '23'), ('PLUS', '+'),
|
||
# ('NUMS', '42'), ('TIMES', '*'), ('NUMS', '10')]
|
||
# 首先，我们要对每种情况编写一个捕获组
|
||
NAME = r'(?P<NAME>[a-zA-Z_][a-zA-Z_0-9]*)'
|
||
NUM = r'(?P<NUM>\d+)'
|
||
PLUS = r'(?P<PLUS>\+)'
|
||
TIMES = r'(?P<TIMES>\*)'
|
||
EQ = r'(?P<EQ>=)'
|
||
WS = r'(?P<WS>\s+)'
|
||
# 把这些捕获组静态编译给re模块
|
||
master_pat = re.compile('|'.join([NAME,NUM,PLUS,TIMES,EQ,WS]))
|
||
|
||
# 定义一个数据结构来存储每个部分
|
||
Token = namedtuple('Token', ['type', 'value'])
|
||
|
||
def generate_tokens(pat, text):
|
||
# 调用re.compile的scanner模块来扫描，并用iter模块将其变成可迭代对象进行迭代
|
||
scanner = pat.scanner(text)
|
||
for m in iter(scanner.match, None):
|
||
# 生成结构来存储分词
|
||
yield Token(m.lastgroup, m.group())
|
||
|
||
# 对分词类型进行过滤，去掉空格
|
||
tokens = (tok for tok in generate_tokens(master_pat, text) if tok.type != 'WS')
|
||
for tok in tokens:
|
||
print(tok)
|
||
|
||
|
||
# 看起来很简单是吧，唯一的难点就在compile的时候，
|
||
# 要把大词放前面先搜完再搜小词，保证较长的模式先匹配；比如<=的顺序就应该比<靠前
|
||
|