TidyLib
tidy/src/lexer.c

   /* lexer.c -- Lexer for html parser
     
     (c) 1998-2005 (W3C) MIT, ERCIM, Keio University
     See tidy.h for the copyright notice.
     
     CVS Info :
   
       $Author: arnaud02 $ 
       $Date: 2005/10/13 12:11:01 $ 
      $Revision: 1.173 $ 
  
  */
  
  /*
    Given a file stream fp it returns a sequence of tokens.
  
       GetToken(fp) gets the next token
       UngetToken(fp) provides one level undo
  
    The tags include an attribute list:
  
      - linked list of attribute/value nodes
      - each node has 2 NULL-terminated strings.
      - entities are replaced in attribute values
  
    white space is compacted if not in preformatted mode
    If not in preformatted mode then leading white space
    is discarded and subsequent white space sequences
    compacted to single space characters.
  
    If XmlTags is no then Tag names are folded to upper
    case and attribute names to lower case.
  
   Not yet done:
      -   Doctype subset and marked sections
  */
  
  #include "tidy-int.h"
  #include "lexer.h"
  #include "parser.h"
  #include "entities.h"
  #include "streamio.h"
  #include "message.h"
  #include "tmbstr.h"
  #include "clean.h"
  #include "utf8.h"
  #include "streamio.h"
  
  /* Forward references
  */
  /* swallows closing '>' */
  static AttVal *ParseAttrs( TidyDocImpl* doc, Bool *isempty );
  
  static tmbstr ParseAttribute( TidyDocImpl* doc, Bool* isempty, 
                               Node **asp, Node **php );
  
  static tmbstr ParseValue( TidyDocImpl* doc, ctmbstr name, Bool foldCase,
                           Bool *isempty, int *pdelim );
  
  static Node *ParseDocTypeDecl(TidyDocImpl* doc);
  
  static void AddAttrToList( AttVal** list, AttVal* av );
  
  /* used to classify characters for lexical purposes */
  #define MAP(c) ((unsigned)c < 128 ? lexmap[(unsigned)c] : 0)
  static uint lexmap[128];
  
  #define IsValidXMLAttrName(name) IsValidXMLID(name)
  #define IsValidXMLElemName(name) IsValidXMLID(name)
  
  static struct _doctypes
  {
      uint score;
      uint vers;
      ctmbstr name;
      ctmbstr fpi;
      ctmbstr si;
  } const W3C_Doctypes[] =
  {
    {  2, HT20, "HTML 2.0",               "-//IETF//DTD HTML 2.0//EN",              NULL,                                                       },
    {  2, HT20, "HTML 2.0",               "-//IETF//DTD HTML//EN",                  NULL,                                                       },
    {  2, HT20, "HTML 2.0",               "-//W3C//DTD HTML 2.0//EN",               NULL,                                                       },
    {  1, HT32, "HTML 3.2",               "-//W3C//DTD HTML 3.2//EN",               NULL,                                                       },
    {  1, HT32, "HTML 3.2",               "-//W3C//DTD HTML 3.2 Final//EN",         NULL,                                                       },
    {  1, HT32, "HTML 3.2",               "-//W3C//DTD HTML 3.2 Draft//EN",         NULL,                                                       },
    {  6, H40S, "HTML 4.0 Strict",        "-//W3C//DTD HTML 4.0//EN",               "http://www.w3.org/TR/REC-html40/strict.dtd"                },
    {  8, H40T, "HTML 4.0 Transitional",  "-//W3C//DTD HTML 4.0 Transitional//EN",  "http://www.w3.org/TR/REC-html40/loose.dtd"                 },
    {  7, H40F, "HTML 4.0 Frameset",      "-//W3C//DTD HTML 4.0 Frameset//EN",      "http://www.w3.org/TR/REC-html40/frameset.dtd"              },
    {  3, H41S, "HTML 4.01 Strict",       "-//W3C//DTD HTML 4.01//EN",              "http://www.w3.org/TR/html4/strict.dtd"                     },
    {  5, H41T, "HTML 4.01 Transitional", "-//W3C//DTD HTML 4.01 Transitional//EN", "http://www.w3.org/TR/html4/loose.dtd"                      },
    {  4, H41F, "HTML 4.01 Frameset",     "-//W3C//DTD HTML 4.01 Frameset//EN",     "http://www.w3.org/TR/html4/frameset.dtd"                   },
    {  9, X10S, "XHTML 1.0 Strict",       "-//W3C//DTD XHTML 1.0 Strict//EN",       "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"         },
    { 11, X10T, "XHTML 1.0 Transitional", "-//W3C//DTD XHTML 1.0 Transitional//EN", "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"   },
    { 10, X10F, "XHTML 1.0 Frameset",     "-//W3C//DTD XHTML 1.0 Frameset//EN",     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-frameset.dtd"       },
    { 12, XH11, "XHTML 1.1",              "-//W3C//DTD XHTML 1.1//EN",              "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"              },
    { 13, XB10, "XHTML Basic 1.0",        "-//W3C//DTD XHTML Basic 1.0//EN",        "http://www.w3.org/TR/xhtml-basic/xhtml-basic10.dtd"        },
  
    /* reminder to add XHTML Print 1.0 support, see http://www.w3.org/TR/xhtml-print */
  #if 0
   { 14, XP10, "XHTML Print 1.0",        "-//W3C//DTD XHTML-Print 1.0//EN",         "http://www.w3.org/MarkUp/DTD/xhtml-print10.dtd"           },
   { 14, XP10, "XHTML Print 1.0",        "-//PWG//DTD XHTML-Print 1.0//EN",         "http://www.xhtml-print.org/xhtml-print/xhtml-print10.dtd" },
 #endif
   /* final entry */
   {  0,    0, NULL,                     NULL,                                     NULL                                                        }
 };
 
 int HTMLVersion(TidyDocImpl* doc)
 {
     uint i;
     uint j = 0;
     uint score = 0;
     uint vers = doc->lexer->versions;
     uint dtver = doc->lexer->doctype;
     TidyDoctypeModes dtmode = (TidyDoctypeModes)cfg(doc, TidyDoctypeMode);
     Bool xhtml = (cfgBool(doc, TidyXmlOut) || doc->lexer->isvoyager) &&
                  !cfgBool(doc, TidyHtmlOut);
     Bool html4 = dtmode == TidyDoctypeStrict || dtmode == TidyDoctypeLoose || VERS_FROM40 & dtver;
 
     for (i = 0; W3C_Doctypes[i].name; ++i)
     {
         if ((xhtml && !(VERS_XHTML & W3C_Doctypes[i].vers)) ||
             (html4 && !(VERS_FROM40 & W3C_Doctypes[i].vers)))
             continue;
 
         if (vers & W3C_Doctypes[i].vers &&
             (W3C_Doctypes[i].score < score || !score))
         {
             score = W3C_Doctypes[i].score;
             j = i;
         }
     }
 
     if (score)
         return W3C_Doctypes[j].vers;
 
     return VERS_UNKNOWN;
 }
 
 ctmbstr GetFPIFromVers(uint vers)
 {
     uint i;
 
     for (i = 0; W3C_Doctypes[i].name; ++i)
         if (W3C_Doctypes[i].vers == vers)
             return W3C_Doctypes[i].fpi;
 
     return NULL;
 }
 
 static ctmbstr GetSIFromVers(uint vers)
 {
     uint i;
 
     for (i = 0; W3C_Doctypes[i].name; ++i)
         if (W3C_Doctypes[i].vers == vers)
             return W3C_Doctypes[i].si;
 
     return NULL;
 }
 
 static ctmbstr GetNameFromVers(uint vers)
 {
     uint i;
 
     for (i = 0; W3C_Doctypes[i].name; ++i)
         if (W3C_Doctypes[i].vers == vers)
             return W3C_Doctypes[i].name;
 
     return NULL;
 }
 
 static uint GetVersFromFPI(ctmbstr fpi)
 {
     uint i;
 
     for (i = 0; W3C_Doctypes[i].name; ++i)
         if (tmbstrcasecmp(W3C_Doctypes[i].fpi, fpi) == 0)
             return W3C_Doctypes[i].vers;
 
     return 0;
 }
 
 /* everything is allowed in proprietary version of HTML */
 /* this is handled here rather than in the tag/attr dicts */
 void ConstrainVersion(TidyDocImpl* doc, uint vers)
 {
     doc->lexer->versions &= (vers | VERS_PROPRIETARY);
 }
 
 Bool IsWhite(uint c)
 {
     uint map = MAP(c);
 
     return (map & white)!=0;
 }
 
 Bool IsNewline(uint c)
 {
     uint map = MAP(c);
     return (map & newline)!=0;
 }
 
 Bool IsDigit(uint c)
 {
     uint map;
 
     map = MAP(c);
 
     return (map & digit)!=0;
 }
 
 Bool IsLetter(uint c)
 {
     uint map;
 
     map = MAP(c);
 
     return (map & letter)!=0;
 }
 
 Bool IsNamechar(uint c)
 {
     uint map = MAP(c);
     return (map & namechar)!=0;
 }
 
 Bool IsXMLLetter(uint c)
 {
     return ((c >= 0x41 && c <= 0x5a) ||
         (c >= 0x61 && c <= 0x7a) ||
         (c >= 0xc0 && c <= 0xd6) ||
         (c >= 0xd8 && c <= 0xf6) ||
         (c >= 0xf8 && c <= 0xff) ||
         (c >= 0x100 && c <= 0x131) ||
         (c >= 0x134 && c <= 0x13e) ||
         (c >= 0x141 && c <= 0x148) ||
         (c >= 0x14a && c <= 0x17e) ||
         (c >= 0x180 && c <= 0x1c3) ||
         (c >= 0x1cd && c <= 0x1f0) ||
         (c >= 0x1f4 && c <= 0x1f5) ||
         (c >= 0x1fa && c <= 0x217) ||
         (c >= 0x250 && c <= 0x2a8) ||
         (c >= 0x2bb && c <= 0x2c1) ||
         c == 0x386 ||
         (c >= 0x388 && c <= 0x38a) ||
         c == 0x38c ||
         (c >= 0x38e && c <= 0x3a1) ||
         (c >= 0x3a3 && c <= 0x3ce) ||
         (c >= 0x3d0 && c <= 0x3d6) ||
         c == 0x3da ||
         c == 0x3dc ||
         c == 0x3de ||
         c == 0x3e0 ||
         (c >= 0x3e2 && c <= 0x3f3) ||
         (c >= 0x401 && c <= 0x40c) ||
         (c >= 0x40e && c <= 0x44f) ||
         (c >= 0x451 && c <= 0x45c) ||
         (c >= 0x45e && c <= 0x481) ||
         (c >= 0x490 && c <= 0x4c4) ||
         (c >= 0x4c7 && c <= 0x4c8) ||
         (c >= 0x4cb && c <= 0x4cc) ||
         (c >= 0x4d0 && c <= 0x4eb) ||
         (c >= 0x4ee && c <= 0x4f5) ||
         (c >= 0x4f8 && c <= 0x4f9) ||
         (c >= 0x531 && c <= 0x556) ||
         c == 0x559 ||
         (c >= 0x561 && c <= 0x586) ||
         (c >= 0x5d0 && c <= 0x5ea) ||
         (c >= 0x5f0 && c <= 0x5f2) ||
         (c >= 0x621 && c <= 0x63a) ||
         (c >= 0x641 && c <= 0x64a) ||
         (c >= 0x671 && c <= 0x6b7) ||
         (c >= 0x6ba && c <= 0x6be) ||
         (c >= 0x6c0 && c <= 0x6ce) ||
         (c >= 0x6d0 && c <= 0x6d3) ||
         c == 0x6d5 ||
         (c >= 0x6e5 && c <= 0x6e6) ||
         (c >= 0x905 && c <= 0x939) ||
         c == 0x93d ||
         (c >= 0x958 && c <= 0x961) ||
         (c >= 0x985 && c <= 0x98c) ||
         (c >= 0x98f && c <= 0x990) ||
         (c >= 0x993 && c <= 0x9a8) ||
         (c >= 0x9aa && c <= 0x9b0) ||
         c == 0x9b2 ||
         (c >= 0x9b6 && c <= 0x9b9) ||
         (c >= 0x9dc && c <= 0x9dd) ||
         (c >= 0x9df && c <= 0x9e1) ||
         (c >= 0x9f0 && c <= 0x9f1) ||
         (c >= 0xa05 && c <= 0xa0a) ||
         (c >= 0xa0f && c <= 0xa10) ||
         (c >= 0xa13 && c <= 0xa28) ||
         (c >= 0xa2a && c <= 0xa30) ||
         (c >= 0xa32 && c <= 0xa33) ||
         (c >= 0xa35 && c <= 0xa36) ||
         (c >= 0xa38 && c <= 0xa39) ||
         (c >= 0xa59 && c <= 0xa5c) ||
         c == 0xa5e ||
         (c >= 0xa72 && c <= 0xa74) ||
         (c >= 0xa85 && c <= 0xa8b) ||
         c == 0xa8d ||
         (c >= 0xa8f && c <= 0xa91) ||
         (c >= 0xa93 && c <= 0xaa8) ||
         (c >= 0xaaa && c <= 0xab0) ||
         (c >= 0xab2 && c <= 0xab3) ||
         (c >= 0xab5 && c <= 0xab9) ||
         c == 0xabd ||
         c == 0xae0 ||
         (c >= 0xb05 && c <= 0xb0c) ||
         (c >= 0xb0f && c <= 0xb10) ||
         (c >= 0xb13 && c <= 0xb28) ||
         (c >= 0xb2a && c <= 0xb30) ||
         (c >= 0xb32 && c <= 0xb33) ||
         (c >= 0xb36 && c <= 0xb39) ||
         c == 0xb3d ||
         (c >= 0xb5c && c <= 0xb5d) ||
         (c >= 0xb5f && c <= 0xb61) ||
         (c >= 0xb85 && c <= 0xb8a) ||
         (c >= 0xb8e && c <= 0xb90) ||
         (c >= 0xb92 && c <= 0xb95) ||
         (c >= 0xb99 && c <= 0xb9a) ||
         c == 0xb9c ||
         (c >= 0xb9e && c <= 0xb9f) ||
         (c >= 0xba3 && c <= 0xba4) ||
         (c >= 0xba8 && c <= 0xbaa) ||
         (c >= 0xbae && c <= 0xbb5) ||
         (c >= 0xbb7 && c <= 0xbb9) ||
         (c >= 0xc05 && c <= 0xc0c) ||
         (c >= 0xc0e && c <= 0xc10) ||
         (c >= 0xc12 && c <= 0xc28) ||
         (c >= 0xc2a && c <= 0xc33) ||
         (c >= 0xc35 && c <= 0xc39) ||
         (c >= 0xc60 && c <= 0xc61) ||
         (c >= 0xc85 && c <= 0xc8c) ||
         (c >= 0xc8e && c <= 0xc90) ||
         (c >= 0xc92 && c <= 0xca8) ||
         (c >= 0xcaa && c <= 0xcb3) ||
         (c >= 0xcb5 && c <= 0xcb9) ||
         c == 0xcde ||
         (c >= 0xce0 && c <= 0xce1) ||
         (c >= 0xd05 && c <= 0xd0c) ||
         (c >= 0xd0e && c <= 0xd10) ||
         (c >= 0xd12 && c <= 0xd28) ||
         (c >= 0xd2a && c <= 0xd39) ||
         (c >= 0xd60 && c <= 0xd61) ||
         (c >= 0xe01 && c <= 0xe2e) ||
         c == 0xe30 ||
         (c >= 0xe32 && c <= 0xe33) ||
         (c >= 0xe40 && c <= 0xe45) ||
         (c >= 0xe81 && c <= 0xe82) ||
         c == 0xe84 ||
         (c >= 0xe87 && c <= 0xe88) ||
         c == 0xe8a ||
         c == 0xe8d ||
         (c >= 0xe94 && c <= 0xe97) ||
         (c >= 0xe99 && c <= 0xe9f) ||
         (c >= 0xea1 && c <= 0xea3) ||
         c == 0xea5 ||
         c == 0xea7 ||
         (c >= 0xeaa && c <= 0xeab) ||
         (c >= 0xead && c <= 0xeae) ||
         c == 0xeb0 ||
         (c >= 0xeb2 && c <= 0xeb3) ||
         c == 0xebd ||
         (c >= 0xec0 && c <= 0xec4) ||
         (c >= 0xf40 && c <= 0xf47) ||
         (c >= 0xf49 && c <= 0xf69) ||
         (c >= 0x10a0 && c <= 0x10c5) ||
         (c >= 0x10d0 && c <= 0x10f6) ||
         c == 0x1100 ||
         (c >= 0x1102 && c <= 0x1103) ||
         (c >= 0x1105 && c <= 0x1107) ||
         c == 0x1109 ||
         (c >= 0x110b && c <= 0x110c) ||
         (c >= 0x110e && c <= 0x1112) ||
         c == 0x113c ||
         c == 0x113e ||
         c == 0x1140 ||
         c == 0x114c ||
         c == 0x114e ||
         c == 0x1150 ||
         (c >= 0x1154 && c <= 0x1155) ||
         c == 0x1159 ||
         (c >= 0x115f && c <= 0x1161) ||
         c == 0x1163 ||
         c == 0x1165 ||
         c == 0x1167 ||
         c == 0x1169 ||
         (c >= 0x116d && c <= 0x116e) ||
         (c >= 0x1172 && c <= 0x1173) ||
         c == 0x1175 ||
         c == 0x119e ||
         c == 0x11a8 ||
         c == 0x11ab ||
         (c >= 0x11ae && c <= 0x11af) ||
         (c >= 0x11b7 && c <= 0x11b8) ||
         c == 0x11ba ||
         (c >= 0x11bc && c <= 0x11c2) ||
         c == 0x11eb ||
         c == 0x11f0 ||
         c == 0x11f9 ||
         (c >= 0x1e00 && c <= 0x1e9b) ||
         (c >= 0x1ea0 && c <= 0x1ef9) ||
         (c >= 0x1f00 && c <= 0x1f15) ||
         (c >= 0x1f18 && c <= 0x1f1d) ||
         (c >= 0x1f20 && c <= 0x1f45) ||
         (c >= 0x1f48 && c <= 0x1f4d) ||
         (c >= 0x1f50 && c <= 0x1f57) ||
         c == 0x1f59 ||
         c == 0x1f5b ||
         c == 0x1f5d ||
         (c >= 0x1f5f && c <= 0x1f7d) ||
         (c >= 0x1f80 && c <= 0x1fb4) ||
         (c >= 0x1fb6 && c <= 0x1fbc) ||
         c == 0x1fbe ||
         (c >= 0x1fc2 && c <= 0x1fc4) ||
         (c >= 0x1fc6 && c <= 0x1fcc) ||
         (c >= 0x1fd0 && c <= 0x1fd3) ||
         (c >= 0x1fd6 && c <= 0x1fdb) ||
         (c >= 0x1fe0 && c <= 0x1fec) ||
         (c >= 0x1ff2 && c <= 0x1ff4) ||
         (c >= 0x1ff6 && c <= 0x1ffc) ||
         c == 0x2126 ||
         (c >= 0x212a && c <= 0x212b) ||
         c == 0x212e ||
         (c >= 0x2180 && c <= 0x2182) ||
         (c >= 0x3041 && c <= 0x3094) ||
         (c >= 0x30a1 && c <= 0x30fa) ||
         (c >= 0x3105 && c <= 0x312c) ||
         (c >= 0xac00 && c <= 0xd7a3) ||
         (c >= 0x4e00 && c <= 0x9fa5) ||
         c == 0x3007 ||
         (c >= 0x3021 && c <= 0x3029) ||
         (c >= 0x4e00 && c <= 0x9fa5) ||
         c == 0x3007 ||
         (c >= 0x3021 && c <= 0x3029));
 }
 
 Bool IsXMLNamechar(uint c)
 {
     return (IsXMLLetter(c) ||
         c == '.' || c == '_' ||
         c == ':' || c == '-' ||
         (c >= 0x300 && c <= 0x345) ||
         (c >= 0x360 && c <= 0x361) ||
         (c >= 0x483 && c <= 0x486) ||
         (c >= 0x591 && c <= 0x5a1) ||
         (c >= 0x5a3 && c <= 0x5b9) ||
         (c >= 0x5bb && c <= 0x5bd) ||
         c == 0x5bf ||
         (c >= 0x5c1 && c <= 0x5c2) ||
         c == 0x5c4 ||
         (c >= 0x64b && c <= 0x652) ||
         c == 0x670 ||
         (c >= 0x6d6 && c <= 0x6dc) ||
         (c >= 0x6dd && c <= 0x6df) ||
         (c >= 0x6e0 && c <= 0x6e4) ||
         (c >= 0x6e7 && c <= 0x6e8) ||
         (c >= 0x6ea && c <= 0x6ed) ||
         (c >= 0x901 && c <= 0x903) ||
         c == 0x93c ||
         (c >= 0x93e && c <= 0x94c) ||
         c == 0x94d ||
         (c >= 0x951 && c <= 0x954) ||
         (c >= 0x962 && c <= 0x963) ||
         (c >= 0x981 && c <= 0x983) ||
         c == 0x9bc ||
         c == 0x9be ||
         c == 0x9bf ||
         (c >= 0x9c0 && c <= 0x9c4) ||
         (c >= 0x9c7 && c <= 0x9c8) ||
         (c >= 0x9cb && c <= 0x9cd) ||
         c == 0x9d7 ||
         (c >= 0x9e2 && c <= 0x9e3) ||
         c == 0xa02 ||
         c == 0xa3c ||
         c == 0xa3e ||
         c == 0xa3f ||
         (c >= 0xa40 && c <= 0xa42) ||
         (c >= 0xa47 && c <= 0xa48) ||
         (c >= 0xa4b && c <= 0xa4d) ||
         (c >= 0xa70 && c <= 0xa71) ||
         (c >= 0xa81 && c <= 0xa83) ||
         c == 0xabc ||
         (c >= 0xabe && c <= 0xac5) ||
         (c >= 0xac7 && c <= 0xac9) ||
         (c >= 0xacb && c <= 0xacd) ||
         (c >= 0xb01 && c <= 0xb03) ||
         c == 0xb3c ||
         (c >= 0xb3e && c <= 0xb43) ||
         (c >= 0xb47 && c <= 0xb48) ||
         (c >= 0xb4b && c <= 0xb4d) ||
         (c >= 0xb56 && c <= 0xb57) ||
         (c >= 0xb82 && c <= 0xb83) ||
         (c >= 0xbbe && c <= 0xbc2) ||
         (c >= 0xbc6 && c <= 0xbc8) ||
         (c >= 0xbca && c <= 0xbcd) ||
         c == 0xbd7 ||
         (c >= 0xc01 && c <= 0xc03) ||
         (c >= 0xc3e && c <= 0xc44) ||
         (c >= 0xc46 && c <= 0xc48) ||
         (c >= 0xc4a && c <= 0xc4d) ||
         (c >= 0xc55 && c <= 0xc56) ||
         (c >= 0xc82 && c <= 0xc83) ||
         (c >= 0xcbe && c <= 0xcc4) ||
         (c >= 0xcc6 && c <= 0xcc8) ||
         (c >= 0xcca && c <= 0xccd) ||
         (c >= 0xcd5 && c <= 0xcd6) ||
         (c >= 0xd02 && c <= 0xd03) ||
         (c >= 0xd3e && c <= 0xd43) ||
         (c >= 0xd46 && c <= 0xd48) ||
         (c >= 0xd4a && c <= 0xd4d) ||
         c == 0xd57 ||
         c == 0xe31 ||
         (c >= 0xe34 && c <= 0xe3a) ||
         (c >= 0xe47 && c <= 0xe4e) ||
         c == 0xeb1 ||
         (c >= 0xeb4 && c <= 0xeb9) ||
         (c >= 0xebb && c <= 0xebc) ||
         (c >= 0xec8 && c <= 0xecd) ||
         (c >= 0xf18 && c <= 0xf19) ||
         c == 0xf35 ||
         c == 0xf37 ||
         c == 0xf39 ||
         c == 0xf3e ||
         c == 0xf3f ||
         (c >= 0xf71 && c <= 0xf84) ||
         (c >= 0xf86 && c <= 0xf8b) ||
         (c >= 0xf90 && c <= 0xf95) ||
         c == 0xf97 ||
         (c >= 0xf99 && c <= 0xfad) ||
         (c >= 0xfb1 && c <= 0xfb7) ||
         c == 0xfb9 ||
         (c >= 0x20d0 && c <= 0x20dc) ||
         c == 0x20e1 ||
         (c >= 0x302a && c <= 0x302f) ||
         c == 0x3099 ||
         c == 0x309a ||
         (c >= 0x30 && c <= 0x39) ||
         (c >= 0x660 && c <= 0x669) ||
         (c >= 0x6f0 && c <= 0x6f9) ||
         (c >= 0x966 && c <= 0x96f) ||
         (c >= 0x9e6 && c <= 0x9ef) ||
         (c >= 0xa66 && c <= 0xa6f) ||
         (c >= 0xae6 && c <= 0xaef) ||
         (c >= 0xb66 && c <= 0xb6f) ||
         (c >= 0xbe7 && c <= 0xbef) ||
         (c >= 0xc66 && c <= 0xc6f) ||
         (c >= 0xce6 && c <= 0xcef) ||
         (c >= 0xd66 && c <= 0xd6f) ||
         (c >= 0xe50 && c <= 0xe59) ||
         (c >= 0xed0 && c <= 0xed9) ||
         (c >= 0xf20 && c <= 0xf29) ||
         c == 0xb7 ||
         c == 0x2d0 ||
         c == 0x2d1 ||
         c == 0x387 ||
         c == 0x640 ||
         c == 0xe46 ||
         c == 0xec6 ||
         c == 0x3005 ||
         (c >= 0x3031 && c <= 0x3035) ||
         (c >= 0x309d && c <= 0x309e) ||
         (c >= 0x30fc && c <= 0x30fe));
 }
 
 Bool IsLower(uint c)
 {
     uint map = MAP(c);
 
     return (map & lowercase)!=0;
 }
 
 Bool IsUpper(uint c)
 {
     uint map = MAP(c);
 
     return (map & uppercase)!=0;
 }
 
 uint ToLower(uint c)
 {
     uint map = MAP(c);
 
     if (map & uppercase)
         c += 'a' - 'A';
 
     return c;
 }
 
 uint ToUpper(uint c)
 {
     uint map = MAP(c);
 
     if (map & lowercase)
         c += (uint) ('A' - 'a' );
 
     return c;
 }
 
 char FoldCase( TidyDocImpl* doc, tmbchar c, Bool tocaps )
 {
     if ( !cfgBool(doc, TidyXmlTags) )
     {
         if ( tocaps )
         {
             c = (tmbchar) ToUpper(c);
         }
         else /* force to lower case */
         {
             c = (tmbchar) ToLower(c);
         }
     }
     return c;
 }
 
 
 /*
  return last character in string
  this is useful when trailing quotemark
  is missing on an attribute
 */
 static tmbchar LastChar( tmbstr str )
 {
     if ( str && *str )
     {
         int n = tmbstrlen(str);
         return str[n-1];
     }
     return 0;
 }
 
 /*
    node->type is one of these:
 
     #define TextNode    1
     #define StartTag    2
     #define EndTag      3
     #define StartEndTag 4
 */
 
 Lexer* NewLexer( TidyDocImpl* doc )
 {
     Lexer* lexer = (Lexer*) MemAlloc( sizeof(Lexer) );
 
     if ( lexer != NULL )
     {
         ClearMemory( lexer, sizeof(Lexer) );
 
         lexer->lines = 1;
         lexer->columns = 1;
         lexer->state = LEX_CONTENT;
 
         lexer->versions = (VERS_ALL|VERS_PROPRIETARY);
         lexer->doctype = VERS_UNKNOWN;
         lexer->root = &doc->root;
     }
     return lexer;
 }
 
 Bool EndOfInput( TidyDocImpl* doc )
 {
     assert( doc->docIn != NULL );
     return ( !doc->docIn->pushed && IsEOF(doc->docIn) );
 }
 
 void FreeLexer( TidyDocImpl* doc )
 {
     Lexer *lexer = doc->lexer;
     if ( lexer )
     {
         FreeStyles( doc );
 
         if ( lexer->pushed )
             FreeNode( doc, lexer->token );
 
         while ( lexer->istacksize > 0 )
             PopInline( doc, NULL );
 
         MemFree( lexer->istack );
         MemFree( lexer->lexbuf );
         MemFree( lexer );
         doc->lexer = NULL;
     }
 }
 
 /* Lexer uses bigger memory chunks than pprint as
 ** it must hold the entire input document. not just
 ** the last line or three.
 */
 void AddByte( Lexer *lexer, tmbchar ch )
 {
     if ( lexer->lexsize + 2 >= lexer->lexlength )
     {
         tmbstr buf = NULL;
         uint allocAmt = lexer->lexlength;
         while ( lexer->lexsize + 2 >= allocAmt )
         {
             if ( allocAmt == 0 )
                 allocAmt = 8192;
             else
                 allocAmt *= 2;
         }
         buf = (tmbstr) MemRealloc( lexer->lexbuf, allocAmt );
         if ( buf )
         {
           ClearMemory( buf + lexer->lexlength, 
                        allocAmt - lexer->lexlength );
           lexer->lexbuf = buf;
           lexer->lexlength = allocAmt;
         }
     }
 
     lexer->lexbuf[ lexer->lexsize++ ] = ch;
     lexer->lexbuf[ lexer->lexsize ]   = '\0';  /* debug */
 }
 
 static void ChangeChar( Lexer *lexer, tmbchar c )
 {
     if ( lexer->lexsize > 0 )
     {
         lexer->lexbuf[ lexer->lexsize-1 ] = c;
     }
 }
 
 /* store character c as UTF-8 encoded byte stream */
 void AddCharToLexer( Lexer *lexer, uint c )
 {
     int i, err, count = 0;
     tmbchar buf[10] = {0};
     
     err = EncodeCharToUTF8Bytes( c, buf, NULL, &count );
     if (err)
     {
 #if 0 && defined(_DEBUG)
         fprintf( stderr, "lexer UTF-8 encoding error for U+%x : ", c );
 #endif
         /* replacement character 0xFFFD encoded as UTF-8 */
         buf[0] = (byte) 0xEF;
         buf[1] = (byte) 0xBF;
         buf[2] = (byte) 0xBD;
         count = 3;
     }
     
     for ( i = 0; i < count; ++i )
         AddByte( lexer, buf[i] );
 }
 
 static void AddStringToLexer( Lexer *lexer, ctmbstr str )
 {
     uint c;
 
     /*  Many (all?) compilers will sign-extend signed chars (the default) when
     **  converting them to unsigned integer values.  We must cast our char to
     **  unsigned char before assigning it to prevent this from happening.
     */
     while( 0 != (c = (unsigned char) *str++ ))
         AddCharToLexer( lexer, c );
 }
 
 /*
   No longer attempts to insert missing ';' for unknown
   enitities unless one was present already, since this
   gives unexpected results.
 
   For example:   <a href="something.htm?foo&bar&fred">
   was tidied to: <a href="something.htm?foo&amp;bar;&amp;fred;">
   rather than:   <a href="something.htm?foo&amp;bar&amp;fred">
 
   My thanks for Maurice Buxton for spotting this.
 
   Also Randy Waki pointed out the following case for the
   04 Aug 00 version (bug #433012):
   
   For example:   <a href="something.htm?id=1&lang=en">
   was tidied to: <a href="something.htm?id=1&lang;=en">
   rather than:   <a href="something.htm?id=1&amp;lang=en">
   
   where "lang" is a known entity (#9001), but browsers would
   misinterpret "&lang;" because it had a value > 256.
   
   So the case of an apparently known entity with a value > 256 and
   missing a semicolon is handled specially.
   
   "ParseEntity" is also a bit of a misnomer - it handles entities and
   numeric character references. Invalid NCR's are now reported.
 */
 static void ParseEntity( TidyDocImpl* doc, int mode )
 {
     uint start;
     Bool first = yes, semicolon = no, found = no;
     Bool isXml = cfgBool( doc, TidyXmlTags );
     uint c, ch, startcol, entver = 0;
     Lexer* lexer = doc->lexer;
 
     start = lexer->lexsize - 1;  /* to start at "&" */
     startcol = doc->docIn->curcol - 1;
 
     while ( (c = ReadChar(doc->docIn)) != EndOfStream )
     {
         if ( c == ';' )
         {
             semicolon = yes;
             break;
         }
 
         if (first && c == '#')
         {
 #if SUPPORT_ASIAN_ENCODINGS
             if ( !cfgBool(doc, TidyNCR) || 
                  cfg(doc, TidyInCharEncoding) == BIG5 ||
                  cfg(doc, TidyInCharEncoding) == SHIFTJIS )
             {
                 UngetChar('#', doc->docIn);
                 return;
             }
 #endif
             AddCharToLexer( lexer, c );
             first = no;
             continue;
         }
 
         first = no;
 
         if ( IsNamechar(c) )
         {
             AddCharToLexer( lexer, c );
             continue;
         }
 
         /* otherwise put it back */
 
         UngetChar( c, doc->docIn );
         break;
     }
 
     /* make sure entity is NULL terminated */
     lexer->lexbuf[lexer->lexsize] = '\0';
 
     /* Should contrain version to XML/XHTML if &apos; 
     ** is encountered.  But this is not possible with
     ** Tidy's content model bit mask.
     */
     if ( tmbstrcmp(lexer->lexbuf+start, "&apos") == 0
          && !cfgBool(doc, TidyXmlOut)
          && !lexer->isvoyager
          && !cfgBool(doc, TidyXhtmlOut) )
         ReportEntityError( doc, APOS_UNDEFINED, lexer->lexbuf+start, 39 );
 
     /* Lookup entity code and version
     */
     found = EntityInfo( lexer->lexbuf+start, isXml, &ch, &entver );
 
     /* deal with unrecognized or invalid entities */
     /* #433012 - fix by Randy Waki 17 Feb 01 */
     /* report invalid NCR's - Terry Teague 01 Sep 01 */
     if ( !found || (ch >= 128 && ch <= 159) || (ch >= 256 && c != ';') )
     {
         /* set error position just before offending character */
         lexer->lines = doc->docIn->curline;
         lexer->columns = startcol;
 
         if (lexer->lexsize > start + 1)
         {
             if (ch >= 128 && ch <= 159)
             {
                 /* invalid numeric character reference */
                 
                 uint c1 = 0;
                 int replaceMode = DISCARDED_CHAR;
             
                 if ( ReplacementCharEncoding == WIN1252 )
                     c1 = DecodeWin1252( ch );
                 else if ( ReplacementCharEncoding == MACROMAN )
                     c1 = DecodeMacRoman( ch );
 
                 if ( c1 )
                     replaceMode = REPLACED_CHAR;
                 
                 if ( c != ';' )  /* issue warning if not terminated by ';' */
                     ReportEntityError( doc, MISSING_SEMICOLON_NCR,
                                        lexer->lexbuf+start, c );
  
                 ReportEncodingError(doc, INVALID_NCR, ch, replaceMode == DISCARDED_CHAR);
                 
                 if ( c1 )
                 {
                     /* make the replacement */
                     lexer->lexsize = start;
                     AddCharToLexer( lexer, c1 );
                     semicolon = no;
                 }
                 else
                 {
                     /* discard */
                     lexer->lexsize = start;
                     semicolon = no;
                }
                
             }
             else
                 ReportEntityError( doc, UNKNOWN_ENTITY,
                                    lexer->lexbuf+start, ch );
 
             if (semicolon)
                 AddCharToLexer( lexer, ';' );
         }
         else /* naked & */
             ReportEntityError( doc, UNESCAPED_AMPERSAND,
                                lexer->lexbuf+start, ch );
     }
     else
     {
         if ( c != ';' )    /* issue warning if not terminated by ';' */
         {
             /* set error position just before offending chararcter */
             lexer->lines = doc->docIn->curline;
             lexer->columns = startcol;
             ReportEntityError( doc, MISSING_SEMICOLON, lexer->lexbuf+start, c );
         }
 
         lexer->lexsize = start;
         if ( ch == 160 && (mode & Preformatted) )
             ch = ' ';
         AddCharToLexer( lexer, ch );
 
         if ( ch == '&' && !cfgBool(doc, TidyQuoteAmpersand) )
             AddStringToLexer( lexer, "amp;" );
 
         /* Detect extended vs. basic entities */
         ConstrainVersion( doc, entver );
     }
 }
 
 static tmbchar ParseTagName( TidyDocImpl* doc )
 {
     Lexer *lexer = doc->lexer;
     uint c = lexer->lexbuf[ lexer->txtstart ];
     Bool xml = cfgBool(doc, TidyXmlTags);
 
     /* fold case of first character in buffer */
     if (!xml && IsUpper(c))
         lexer->lexbuf[lexer->txtstart] = (tmbchar) ToLower(c);
 
     while ((c = ReadChar(doc->docIn)) != EndOfStream)
     {
         if ((!xml && !IsNamechar(c)) ||
             (xml && !IsXMLNamechar(c)))
             break;
 
         /* fold case of subsequent characters */
         if (!xml && IsUpper(c))
              c = ToLower(c);
 
         AddCharToLexer(lexer, c);
     }
 
     lexer->txtend = lexer->lexsize;
     return (tmbchar) c;
 }
 
 /*
   Used for elements and text nodes
   element name is NULL for text nodes
   start and end are offsets into lexbuf
   which contains the textual content of
   all elements in the parse tree.
 
   parent and content allow traversal
   of the parse tree in any direction.
   attributes are represented as a linked
   list of AttVal nodes which hold the
   strings for attribute/value pairs.
 */
 
 
 Node *NewNode(Lexer *lexer)
 {
     Node* node = (Node*) MemAlloc( sizeof(Node) );
     ClearMemory( node, sizeof(Node) );
     if ( lexer )
     {
         node->line = lexer->lines;
         node->column = lexer->columns;
     }
     node->type = TextNode;
     return node;
 }
 
 /* used to clone heading nodes when split by an <HR> */
 Node *CloneNode( TidyDocImpl* doc, Node *element )
 {
     Lexer* lexer = doc->lexer;
     Node *node = NewNode( lexer );
 
     node->start = lexer->lexsize;
     node->end   = lexer->lexsize;
 
     if ( element )
     {
         node->parent     = element->parent;
         node->type       = element->type;
         node->closed     = element->closed;
         node->implicit   = element->implicit;
         node->tag        = element->tag;
         node->element    = tmbstrdup( element->element );
         node->attributes = DupAttrs( doc, element->attributes );
     }
     return node;
 }
 
 /* free node's attributes */
 void FreeAttrs( TidyDocImpl* doc, Node *node )
 {
 
     while ( node->attributes )
     {
         AttVal *av = node->attributes;
 
         if ( av->attribute )
         {
             if ( (attrIsID(av) || attrIsNAME(av)) &&
                  IsAnchorElement(doc, node) )
             {
                 RemoveAnchorByNode( doc, node );
             }
         }
 
         node->attributes = av->next;
         FreeAttribute( doc, av );
     }
 }
 
 /* doesn't repair attribute list linkage */
 void FreeAttribute( TidyDocImpl* doc, AttVal *av )
 {
     FreeNode( doc, av->asp );
     FreeNode( doc, av->php );
     MemFree( av->attribute );
     MemFree( av->value );
     MemFree( av );
 }
 
 /* detach attribute from node
 */
 void DetachAttribute( Node *node, AttVal *attr )
 {
     AttVal *av, *prev = NULL;
 
     for ( av = node->attributes; av; av = av->next )
     {
         if ( av == attr )
         {
             if ( prev )
                 prev->next = attr->next;
             else
                 node->attributes = attr->next;
             break;
         }
         prev = av;
     }
 }
 
 /* detach attribute from node then free it
 */
 void RemoveAttribute( TidyDocImpl* doc, Node *node, AttVal *attr )
 {
     DetachAttribute( node, attr );
     FreeAttribute( doc, attr );
 }
 
 /*
   Free document nodes by iterating through peers and recursing
   through children. Set next to NULL before calling FreeNode()
   to avoid freeing peer nodes. Doesn't patch up prev/next links.
  */
 void FreeNode( TidyDocImpl* doc, Node *node )
 {
     while ( node )
     {
         Node* next = node->next;
 
         FreeAttrs( doc, node );
         FreeNode( doc, node->content );
         MemFree( node->element );
 #ifdef TIDY_STORE_ORIGINAL_TEXT
         if (node->otext)
             MemFree(node->otext);
 #endif
         if (RootNode != node->type)
             MemFree( node );
         else
             node->content = NULL;
 
         node = next;
     }
 }
 
 #ifdef TIDY_STORE_ORIGINAL_TEXT
 void StoreOriginalTextInToken(TidyDocImpl* doc, Node* node, uint count)
 {
     if (!doc->storeText)
         return;
 
     if (count >= doc->docIn->otextlen)
         return;
 
     if (!doc->docIn->otextsize)
         return;
 
     if (count == 0)
     {
         node->otext = doc->docIn->otextbuf;
         doc->docIn->otextbuf = NULL;
         doc->docIn->otextlen = 0;
         doc->docIn->otextsize = 0;
     }
     else
     {
         uint len = doc->docIn->otextlen;
         tmbstr buf1 = (tmbstr)MemAlloc(len - count + 1);
         tmbstr buf2 = (tmbstr)MemAlloc(count + 1);
         uint i, j;
 
         /* strncpy? */
 
         for (i = 0; i < len - count; ++i)
             buf1[i] = doc->docIn->otextbuf[i];
 
         buf1[i] = 0;
 
         for (j = 0; j + i < len; ++j)
             buf2[j] = doc->docIn->otextbuf[j + i];
 
         buf2[j] = 0;
 
         MemFree(doc->docIn->otextbuf);
         node->otext = buf1;
         doc->docIn->otextbuf = buf2;
         doc->docIn->otextlen = count;
         doc->docIn->otextsize = count + 1;
     }
 }
 #endif
 
 Node* TextToken( Lexer *lexer )
 {
     Node *node = NewNode( lexer );
     node->start = lexer->txtstart;
     node->end = lexer->txtend;
     return node;
 }
 
 /* used for creating preformatted text from Word2000 */
 Node *NewLineNode( Lexer *lexer )
 {
     Node *node = NewNode( lexer );
     node->start = lexer->lexsize;
     AddCharToLexer( lexer, (uint)'\n' );
     node->end = lexer->lexsize;
     return node;
 }
 
 /* used for adding a &nbsp; for Word2000 */
 Node* NewLiteralTextNode( Lexer *lexer, ctmbstr txt )
 {
     Node *node = NewNode( lexer );
     node->start = lexer->lexsize;
     AddStringToLexer( lexer, txt );
     node->end = lexer->lexsize;
     return node;
 }
 
 static Node* TagToken( TidyDocImpl* doc, NodeType type )
 {
     Lexer* lexer = doc->lexer;
     Node* node = NewNode( lexer );
     node->type = type;
     node->element = tmbstrndup( lexer->lexbuf + lexer->txtstart,
                                 lexer->txtend - lexer->txtstart );
     node->start = lexer->txtstart;
     node->end = lexer->txtstart;
 
     if ( type == StartTag || type == StartEndTag || type == EndTag )
         FindTag(doc, node);
 
     return node;
 }
 
 static Node* NewToken(TidyDocImpl* doc, NodeType type)
 {
     Lexer* lexer = doc->lexer;
     Node* node = NewNode(lexer);
     node->type = type;
     node->start = lexer->txtstart;
     node->end = lexer->txtend;
 #ifdef TIDY_STORE_ORIGINAL_TEXT
     StoreOriginalTextInToken(doc, node, 0);
 #endif
     return node;
 }
 
 #define CommentToken(doc) NewToken(doc, CommentTag)
 #define DocTypeToken(doc) NewToken(doc, DocTypeTag)
 #define PIToken(doc)      NewToken(doc, ProcInsTag)
 #define AspToken(doc)     NewToken(doc, AspTag)
 #define JsteToken(doc)    NewToken(doc, JsteTag)
 #define PhpToken(doc)     NewToken(doc, PhpTag)
 #define XmlDeclToken(doc) NewToken(doc, XmlDecl)
 #define SectionToken(doc) NewToken(doc, SectionTag)
 #define CDATAToken(doc)   NewToken(doc, CDATATag)
 
 void AddStringLiteral( Lexer* lexer, ctmbstr str )
 {
     byte c;
     while(0 != (c = *str++) )
         AddCharToLexer( lexer, c );
 }
 
 void AddStringLiteralLen( Lexer* lexer, ctmbstr str, int len )
 {
     byte c;
     int ix;
 
     for ( ix=0; ix < len && (c = *str++); ++ix )
         AddCharToLexer(lexer, c);
 }
 
 /* find doctype element */
 Node *FindDocType( TidyDocImpl* doc )
 {
     Node* node;
     for ( node = (doc ? doc->root.content : NULL);
           node && node->type != DocTypeTag; 
           node = node->next )
         /**/;
     return node;
 }
 
 /* find parent container element */
 Node* FindContainer( Node* node )
 {
     for ( node = (node ? node->parent : NULL);
           node && nodeHasCM(node, CM_INLINE);
           node = node->parent )
         /**/;
 
     return node;
 }
 
 
 /* find html element */
 Node *FindHTML( TidyDocImpl* doc )
 {
     Node *node;
     for ( node = (doc ? doc->root.content : NULL);
           node && !nodeIsHTML(node); 
           node = node->next )
         /**/;
 
     return node;
 }
 
 /* find XML Declaration */
 Node *FindXmlDecl(TidyDocImpl* doc)
 {
     Node *node;
     for ( node = (doc ? doc->root.content : NULL);
           node && !(node->type == XmlDecl);
           node = node->next )
         /**/;
 
     return node;
 }
 
 
 Node *FindHEAD( TidyDocImpl* doc )
 {
     Node *node = FindHTML( doc );
 
     if ( node )
     {
         for ( node = node->content;
               node && !nodeIsHEAD(node); 
               node = node->next )
             /**/;
     }
 
     return node;
 }
 
 Node *FindTITLE(TidyDocImpl* doc)
 {
     Node *node = FindHEAD(doc);
 
     if (node)
         for (node = node->content;
              node && !nodeIsTITLE(node);
              node = node->next) {}
 
     return node;
 }
 
 Node *FindBody( TidyDocImpl* doc )
 {
     Node *node = ( doc ? doc->root.content : NULL );
 
     while ( node && !nodeIsHTML(node) )
         node = node->next;
 
     if (node == NULL)
         return NULL;
 
     node = node->content;
     while ( node && !nodeIsBODY(node) && !nodeIsFRAMESET(node) )
         node = node->next;
 
     if ( node && nodeIsFRAMESET(node) )
     {
         node = node->content;
         while ( node && !nodeIsNOFRAMES(node) )
             node = node->next;
 
         if ( node )
         {
             node = node->content;
             while ( node && !nodeIsBODY(node) )
                 node = node->next;
         }
     }
 
     return node;
 }
 
 /* add meta element for Tidy */
 Bool AddGenerator( TidyDocImpl* doc )
 {
     AttVal *attval;
     Node *node;
     Node *head = FindHEAD( doc );
     tmbchar buf[256];
     
     if (head)
     {
 #ifdef PLATFORM_NAME
         tmbsnprintf(buf, sizeof(buf), "HTML Tidy for "PLATFORM_NAME" (vers %s), see www.w3.org",
                  tidyReleaseDate());
 #else
         tmbsnprintf(buf, sizeof(buf), "HTML Tidy (vers %s), see www.w3.org", tidyReleaseDate());
 #endif
 
         for ( node = head->content; node; node = node->next )
         {
             if ( nodeIsMETA(node) )
             {