import re
class recollector:
def __init__(self):
self.res={}
def add(self, name, reg ):
re.compile(reg) # check that it is valid
self.res[name] = reg % self.res
collector = recollector()
a = collector.add
a("TextSE" , "[^<]+")
a("UntilHyphen" , "[^-]*-")
a("Until2Hyphens" , "%(UntilHyphen)s(?:[^-]%(UntilHyphen)s)*-")
a("CommentCE" , "%(Until2Hyphens)s>?")
a("UntilRSBs" , "[^\\]]*](?:[^\\]]+])*]+")
a("CDATA_CE" , "%(UntilRSBs)s(?:[^\\]>]%(UntilRSBs)s)*>" )
a("S" , "[ \\n\\t\\r]+")
a("NameStrt" , "[A-Za-z_:]|[^\\x00-\\x7F]")
a("NameChar" , "[A-Za-z0-9_:.-]|[^\\x00-\\x7F]")
a("Name" , "(?:%(NameStrt)s)(?:%(NameChar)s)*")
a("QuoteSE" , "\"[^\"]*\"|'[^']*'")
a("DT_IdentSE" , "%(S)s%(Name)s(?:%(S)s(?:%(Name)s|%(QuoteSE)s))*" )
a("MarkupDeclCE" , "(?:[^\\]\"'><]+|%(QuoteSE)s)*>" )
a("S1" , "[\\n\\r\\t ]")
a("UntilQMs" , "[^?]*\\?+")
a("PI_Tail" , "\\?>|%(S1)s%(UntilQMs)s(?:[^>?]%(UntilQMs)s)*>" )
a("DT_ItemSE" ,
"<(?:!(?:--%(Until2Hyphens)s>|[^-]%(MarkupDeclCE)s)|\\?%(Name)s(?:%(PI_Tail)s))|%%%(Name)s;|%(S)s"
)
a("DocTypeCE" ,
"%(DT_IdentSE)s(?:%(S)s)?(?:\\[(?:%(DT_ItemSE)s)*](?:%(S)s)?)?>?" )
a("DeclCE" ,
"--(?:%(CommentCE)s)?|\\[CDATA\\[(?:%(CDATA_CE)s)?|DOCTYPE(?:%(DocTypeCE)s)?")
a("PI_CE" , "%(Name)s(?:%(PI_Tail)s)?")
a("EndTagCE" , "%(Name)s(?:%(S)s)?>?")
a("AttValSE" , "\"[^<\"]*\"|'[^<']*'")
a("ElemTagCE" ,
"%(Name)s(?:%(S)s%(Name)s(?:%(S)s)?=(?:%(S)s)?(?:%(AttValSE)s))*(?:%(S)s)?/?>?")
a("MarkupSPE" ,
"<(?:!(?:%(DeclCE)s)?|\\?(?:%(PI_CE)s)?|/(?:%(EndTagCE)s)?|(?:%(ElemTagCE)s)?)")
a("XML_SPE" , "%(TextSE)s|%(MarkupSPE)s")
a("XML_MARKUP_ONLY_SPE" , "%(MarkupSPE)s")
def lexxml(data, markuponly=0):
if markuponly:
reg = "XML_MARKUP_ONLY_SPE"
else:
reg = "XML_SPE"
regex = re.compile(collector.res[reg])
return regex.findall(data)
def assertlex(data, numtokens, markuponly=0):
tokens = lexxml(data, markuponly)
if len(tokens)!=numtokens:
assert len(lexxml(data))==numtokens, "data = '%s', numtokens = '%s'" %(data, numotkens)
if not markuponly:
assert "".join(tokens)==data
walktokens(tokens)
def walktokens(tokens):
print
for token in tokens:
if token.startswith("<"):
if token.startswith("<!"):
print "declaration:", token
elif token.startswith("<?xml"):
print "xml declaration:", token
elif token.startswith("<?"):
print "processing instruction:", token
elif token.startswith("</"):
print "end-tag:", token
elif token.endswith("/>"):
print "empty-tag:", token
elif token.endswith(">"):
print "start-tag:", token
else:
print "error:", token
else:
print "text:", token
def testlexer():
# this test suite could be larger!
assertlex("<abc/>", 1)
assertlex("<abc><def/></abc>", 3)
assertlex("<abc>Blah</abc>", 3)
assertlex("<abc>Blah</abc>", 2, markuponly=1)
assertlex("<?xml version='1.0'?><abc>Blah</abc>", 3, markuponly=1)
assertlex("<abc>Blah&foo;Blah</abc>", 3)
assertlex("<abc>Blah&foo;Blah</abc>", 2, markuponly=1)
assertlex("<abc><abc>", 2)
assertlex("</abc></abc>", 2)
assertlex("<abc></def></abc>", 3)
if __name__=="__main__":
testlexer()
