llex.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578
  1. /*
  2. ** $Id: llex.c $
  3. ** Lexical Analyzer
  4. ** See Copyright Notice in lua.h
  5. */
  6. #define llex_c
  7. #define LUA_CORE
  8. #include "lprefix.h"
  9. #include <locale.h>
  10. #include <string.h>
  11. #include "lua.h"
  12. #include "lctype.h"
  13. #include "ldebug.h"
  14. #include "ldo.h"
  15. #include "lgc.h"
  16. #include "llex.h"
  17. #include "lobject.h"
  18. #include "lparser.h"
  19. #include "lstate.h"
  20. #include "lstring.h"
  21. #include "ltable.h"
  22. #include "lzio.h"
  23. #define next(ls) (ls->current = zgetc(ls->z))
  24. #define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r')
  25. /* ORDER RESERVED */
  26. static const char *const luaX_tokens [] = {
  27. "and", "break", "do", "else", "elseif",
  28. "end", "false", "for", "function", "goto", "if",
  29. "in", "local", "nil", "not", "or", "repeat",
  30. "return", "then", "true", "until", "while",
  31. "//", "..", "...", "==", ">=", "<=", "~=",
  32. "<<", ">>", "::", "<eof>",
  33. "<number>", "<integer>", "<name>", "<string>"
  34. };
  35. #define save_and_next(ls) (save(ls, ls->current), next(ls))
  36. static l_noret lexerror (LexState *ls, const char *msg, int token);
  37. static void save (LexState *ls, int c) {
  38. Mbuffer *b = ls->buff;
  39. if (luaZ_bufflen(b) + 1 > luaZ_sizebuffer(b)) {
  40. size_t newsize;
  41. if (luaZ_sizebuffer(b) >= MAX_SIZE/2)
  42. lexerror(ls, "lexical element too long", 0);
  43. newsize = luaZ_sizebuffer(b) * 2;
  44. luaZ_resizebuffer(ls->L, b, newsize);
  45. }
  46. b->buffer[luaZ_bufflen(b)++] = cast_char(c);
  47. }
  48. void luaX_init (lua_State *L) {
  49. int i;
  50. TString *e = luaS_newliteral(L, LUA_ENV); /* create env name */
  51. luaC_fix(L, obj2gco(e)); /* never collect this name */
  52. for (i=0; i<NUM_RESERVED; i++) {
  53. TString *ts = luaS_new(L, luaX_tokens[i]);
  54. luaC_fix(L, obj2gco(ts)); /* reserved words are never collected */
  55. ts->extra = cast_byte(i+1); /* reserved word */
  56. }
  57. }
  58. const char *luaX_token2str (LexState *ls, int token) {
  59. if (token < FIRST_RESERVED) { /* single-byte symbols? */
  60. if (lisprint(token))
  61. return luaO_pushfstring(ls->L, "'%c'", token);
  62. else /* control character */
  63. return luaO_pushfstring(ls->L, "'<\\%d>'", token);
  64. }
  65. else {
  66. const char *s = luaX_tokens[token - FIRST_RESERVED];
  67. if (token < TK_EOS) /* fixed format (symbols and reserved words)? */
  68. return luaO_pushfstring(ls->L, "'%s'", s);
  69. else /* names, strings, and numerals */
  70. return s;
  71. }
  72. }
  73. static const char *txtToken (LexState *ls, int token) {
  74. switch (token) {
  75. case TK_NAME: case TK_STRING:
  76. case TK_FLT: case TK_INT:
  77. save(ls, '\0');
  78. return luaO_pushfstring(ls->L, "'%s'", luaZ_buffer(ls->buff));
  79. default:
  80. return luaX_token2str(ls, token);
  81. }
  82. }
  83. static l_noret lexerror (LexState *ls, const char *msg, int token) {
  84. msg = luaG_addinfo(ls->L, msg, ls->source, ls->linenumber);
  85. if (token)
  86. luaO_pushfstring(ls->L, "%s near %s", msg, txtToken(ls, token));
  87. luaD_throw(ls->L, LUA_ERRSYNTAX);
  88. }
  89. l_noret luaX_syntaxerror (LexState *ls, const char *msg) {
  90. lexerror(ls, msg, ls->t.token);
  91. }
  92. /*
  93. ** creates a new string and anchors it in scanner's table so that
  94. ** it will not be collected until the end of the compilation
  95. ** (by that time it should be anchored somewhere)
  96. */
  97. TString *luaX_newstring (LexState *ls, const char *str, size_t l) {
  98. lua_State *L = ls->L;
  99. TValue *o; /* entry for 'str' */
  100. TString *ts = luaS_newlstr(L, str, l); /* create new string */
  101. setsvalue2s(L, L->top++, ts); /* temporarily anchor it in stack */
  102. o = luaH_set(L, ls->h, s2v(L->top - 1));
  103. if (isempty(o)) { /* not in use yet? */
  104. /* boolean value does not need GC barrier;
  105. table is not a metatable, so it does not need to invalidate cache */
  106. setbtvalue(o); /* t[string] = true */
  107. luaC_checkGC(L);
  108. }
  109. else { /* string already present */
  110. ts = keystrval(nodefromval(o)); /* re-use value previously stored */
  111. }
  112. L->top--; /* remove string from stack */
  113. return ts;
  114. }
  115. /*
  116. ** increment line number and skips newline sequence (any of
  117. ** \n, \r, \n\r, or \r\n)
  118. */
  119. static void inclinenumber (LexState *ls) {
  120. int old = ls->current;
  121. lua_assert(currIsNewline(ls));
  122. next(ls); /* skip '\n' or '\r' */
  123. if (currIsNewline(ls) && ls->current != old)
  124. next(ls); /* skip '\n\r' or '\r\n' */
  125. if (++ls->linenumber >= MAX_INT)
  126. lexerror(ls, "chunk has too many lines", 0);
  127. }
  128. void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source,
  129. int firstchar) {
  130. ls->t.token = 0;
  131. ls->L = L;
  132. ls->current = firstchar;
  133. ls->lookahead.token = TK_EOS; /* no look-ahead token */
  134. ls->z = z;
  135. ls->fs = NULL;
  136. ls->linenumber = 1;
  137. ls->lastline = 1;
  138. ls->source = source;
  139. ls->envn = luaS_newliteral(L, LUA_ENV); /* get env name */
  140. luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER); /* initialize buffer */
  141. }
  142. /*
  143. ** =======================================================
  144. ** LEXICAL ANALYZER
  145. ** =======================================================
  146. */
  147. static int check_next1 (LexState *ls, int c) {
  148. if (ls->current == c) {
  149. next(ls);
  150. return 1;
  151. }
  152. else return 0;
  153. }
  154. /*
  155. ** Check whether current char is in set 'set' (with two chars) and
  156. ** saves it
  157. */
  158. static int check_next2 (LexState *ls, const char *set) {
  159. lua_assert(set[2] == '\0');
  160. if (ls->current == set[0] || ls->current == set[1]) {
  161. save_and_next(ls);
  162. return 1;
  163. }
  164. else return 0;
  165. }
  166. /* LUA_NUMBER */
  167. /*
  168. ** This function is quite liberal in what it accepts, as 'luaO_str2num'
  169. ** will reject ill-formed numerals. Roughly, it accepts the following
  170. ** pattern:
  171. **
  172. ** %d(%x|%.|([Ee][+-]?))* | 0[Xx](%x|%.|([Pp][+-]?))*
  173. **
  174. ** The only tricky part is to accept [+-] only after a valid exponent
  175. ** mark, to avoid reading '3-4' or '0xe+1' as a single number.
  176. **
  177. ** The caller might have already read an initial dot.
  178. */
  179. static int read_numeral (LexState *ls, SemInfo *seminfo) {
  180. TValue obj;
  181. const char *expo = "Ee";
  182. int first = ls->current;
  183. lua_assert(lisdigit(ls->current));
  184. save_and_next(ls);
  185. if (first == '0' && check_next2(ls, "xX")) /* hexadecimal? */
  186. expo = "Pp";
  187. for (;;) {
  188. if (check_next2(ls, expo)) /* exponent mark? */
  189. check_next2(ls, "-+"); /* optional exponent sign */
  190. else if (lisxdigit(ls->current) || ls->current == '.') /* '%x|%.' */
  191. save_and_next(ls);
  192. else break;
  193. }
  194. if (lislalpha(ls->current)) /* is numeral touching a letter? */
  195. save_and_next(ls); /* force an error */
  196. save(ls, '\0');
  197. if (luaO_str2num(luaZ_buffer(ls->buff), &obj) == 0) /* format error? */
  198. lexerror(ls, "malformed number", TK_FLT);
  199. if (ttisinteger(&obj)) {
  200. seminfo->i = ivalue(&obj);
  201. return TK_INT;
  202. }
  203. else {
  204. lua_assert(ttisfloat(&obj));
  205. seminfo->r = fltvalue(&obj);
  206. return TK_FLT;
  207. }
  208. }
  209. /*
  210. ** read a sequence '[=*[' or ']=*]', leaving the last bracket. If
  211. ** sequence is well formed, return its number of '='s + 2; otherwise,
  212. ** return 1 if it is a single bracket (no '='s and no 2nd bracket);
  213. ** otherwise (an unfinished '[==...') return 0.
  214. */
  215. static size_t skip_sep (LexState *ls) {
  216. size_t count = 0;
  217. int s = ls->current;
  218. lua_assert(s == '[' || s == ']');
  219. save_and_next(ls);
  220. while (ls->current == '=') {
  221. save_and_next(ls);
  222. count++;
  223. }
  224. return (ls->current == s) ? count + 2
  225. : (count == 0) ? 1
  226. : 0;
  227. }
  228. static void read_long_string (LexState *ls, SemInfo *seminfo, size_t sep) {
  229. int line = ls->linenumber; /* initial line (for error message) */
  230. save_and_next(ls); /* skip 2nd '[' */
  231. if (currIsNewline(ls)) /* string starts with a newline? */
  232. inclinenumber(ls); /* skip it */
  233. for (;;) {
  234. switch (ls->current) {
  235. case EOZ: { /* error */
  236. const char *what = (seminfo ? "string" : "comment");
  237. const char *msg = luaO_pushfstring(ls->L,
  238. "unfinished long %s (starting at line %d)", what, line);
  239. lexerror(ls, msg, TK_EOS);
  240. break; /* to avoid warnings */
  241. }
  242. case ']': {
  243. if (skip_sep(ls) == sep) {
  244. save_and_next(ls); /* skip 2nd ']' */
  245. goto endloop;
  246. }
  247. break;
  248. }
  249. case '\n': case '\r': {
  250. save(ls, '\n');
  251. inclinenumber(ls);
  252. if (!seminfo) luaZ_resetbuffer(ls->buff); /* avoid wasting space */
  253. break;
  254. }
  255. default: {
  256. if (seminfo) save_and_next(ls);
  257. else next(ls);
  258. }
  259. }
  260. } endloop:
  261. if (seminfo)
  262. seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + sep,
  263. luaZ_bufflen(ls->buff) - 2 * sep);
  264. }
  265. static void esccheck (LexState *ls, int c, const char *msg) {
  266. if (!c) {
  267. if (ls->current != EOZ)
  268. save_and_next(ls); /* add current to buffer for error message */
  269. lexerror(ls, msg, TK_STRING);
  270. }
  271. }
  272. static int gethexa (LexState *ls) {
  273. save_and_next(ls);
  274. esccheck (ls, lisxdigit(ls->current), "hexadecimal digit expected");
  275. return luaO_hexavalue(ls->current);
  276. }
  277. static int readhexaesc (LexState *ls) {
  278. int r = gethexa(ls);
  279. r = (r << 4) + gethexa(ls);
  280. luaZ_buffremove(ls->buff, 2); /* remove saved chars from buffer */
  281. return r;
  282. }
  283. static unsigned long readutf8esc (LexState *ls) {
  284. unsigned long r;
  285. int i = 4; /* chars to be removed: '\', 'u', '{', and first digit */
  286. save_and_next(ls); /* skip 'u' */
  287. esccheck(ls, ls->current == '{', "missing '{'");
  288. r = gethexa(ls); /* must have at least one digit */
  289. while (cast_void(save_and_next(ls)), lisxdigit(ls->current)) {
  290. i++;
  291. esccheck(ls, r <= (0x7FFFFFFFu >> 4), "UTF-8 value too large");
  292. r = (r << 4) + luaO_hexavalue(ls->current);
  293. }
  294. esccheck(ls, ls->current == '}', "missing '}'");
  295. next(ls); /* skip '}' */
  296. luaZ_buffremove(ls->buff, i); /* remove saved chars from buffer */
  297. return r;
  298. }
  299. static void utf8esc (LexState *ls) {
  300. char buff[UTF8BUFFSZ];
  301. int n = luaO_utf8esc(buff, readutf8esc(ls));
  302. for (; n > 0; n--) /* add 'buff' to string */
  303. save(ls, buff[UTF8BUFFSZ - n]);
  304. }
  305. static int readdecesc (LexState *ls) {
  306. int i;
  307. int r = 0; /* result accumulator */
  308. for (i = 0; i < 3 && lisdigit(ls->current); i++) { /* read up to 3 digits */
  309. r = 10*r + ls->current - '0';
  310. save_and_next(ls);
  311. }
  312. esccheck(ls, r <= UCHAR_MAX, "decimal escape too large");
  313. luaZ_buffremove(ls->buff, i); /* remove read digits from buffer */
  314. return r;
  315. }
  316. static void read_string (LexState *ls, int del, SemInfo *seminfo) {
  317. save_and_next(ls); /* keep delimiter (for error messages) */
  318. while (ls->current != del) {
  319. switch (ls->current) {
  320. case EOZ:
  321. lexerror(ls, "unfinished string", TK_EOS);
  322. break; /* to avoid warnings */
  323. case '\n':
  324. case '\r':
  325. lexerror(ls, "unfinished string", TK_STRING);
  326. break; /* to avoid warnings */
  327. case '\\': { /* escape sequences */
  328. int c; /* final character to be saved */
  329. save_and_next(ls); /* keep '\\' for error messages */
  330. switch (ls->current) {
  331. case 'a': c = '\a'; goto read_save;
  332. case 'b': c = '\b'; goto read_save;
  333. case 'f': c = '\f'; goto read_save;
  334. case 'n': c = '\n'; goto read_save;
  335. case 'r': c = '\r'; goto read_save;
  336. case 't': c = '\t'; goto read_save;
  337. case 'v': c = '\v'; goto read_save;
  338. case 'x': c = readhexaesc(ls); goto read_save;
  339. case 'u': utf8esc(ls); goto no_save;
  340. case '\n': case '\r':
  341. inclinenumber(ls); c = '\n'; goto only_save;
  342. case '\\': case '\"': case '\'':
  343. c = ls->current; goto read_save;
  344. case EOZ: goto no_save; /* will raise an error next loop */
  345. case 'z': { /* zap following span of spaces */
  346. luaZ_buffremove(ls->buff, 1); /* remove '\\' */
  347. next(ls); /* skip the 'z' */
  348. while (lisspace(ls->current)) {
  349. if (currIsNewline(ls)) inclinenumber(ls);
  350. else next(ls);
  351. }
  352. goto no_save;
  353. }
  354. default: {
  355. esccheck(ls, lisdigit(ls->current), "invalid escape sequence");
  356. c = readdecesc(ls); /* digital escape '\ddd' */
  357. goto only_save;
  358. }
  359. }
  360. read_save:
  361. next(ls);
  362. /* go through */
  363. only_save:
  364. luaZ_buffremove(ls->buff, 1); /* remove '\\' */
  365. save(ls, c);
  366. /* go through */
  367. no_save: break;
  368. }
  369. default:
  370. save_and_next(ls);
  371. }
  372. }
  373. save_and_next(ls); /* skip delimiter */
  374. seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1,
  375. luaZ_bufflen(ls->buff) - 2);
  376. }
  377. static int llex (LexState *ls, SemInfo *seminfo) {
  378. luaZ_resetbuffer(ls->buff);
  379. for (;;) {
  380. switch (ls->current) {
  381. case '\n': case '\r': { /* line breaks */
  382. inclinenumber(ls);
  383. break;
  384. }
  385. case ' ': case '\f': case '\t': case '\v': { /* spaces */
  386. next(ls);
  387. break;
  388. }
  389. case '-': { /* '-' or '--' (comment) */
  390. next(ls);
  391. if (ls->current != '-') return '-';
  392. /* else is a comment */
  393. next(ls);
  394. if (ls->current == '[') { /* long comment? */
  395. size_t sep = skip_sep(ls);
  396. luaZ_resetbuffer(ls->buff); /* 'skip_sep' may dirty the buffer */
  397. if (sep >= 2) {
  398. read_long_string(ls, NULL, sep); /* skip long comment */
  399. luaZ_resetbuffer(ls->buff); /* previous call may dirty the buff. */
  400. break;
  401. }
  402. }
  403. /* else short comment */
  404. while (!currIsNewline(ls) && ls->current != EOZ)
  405. next(ls); /* skip until end of line (or end of file) */
  406. break;
  407. }
  408. case '[': { /* long string or simply '[' */
  409. size_t sep = skip_sep(ls);
  410. if (sep >= 2) {
  411. read_long_string(ls, seminfo, sep);
  412. return TK_STRING;
  413. }
  414. else if (sep == 0) /* '[=...' missing second bracket? */
  415. lexerror(ls, "invalid long string delimiter", TK_STRING);
  416. return '[';
  417. }
  418. case '=': {
  419. next(ls);
  420. if (check_next1(ls, '=')) return TK_EQ; /* '==' */
  421. else return '=';
  422. }
  423. case '<': {
  424. next(ls);
  425. if (check_next1(ls, '=')) return TK_LE; /* '<=' */
  426. else if (check_next1(ls, '<')) return TK_SHL; /* '<<' */
  427. else return '<';
  428. }
  429. case '>': {
  430. next(ls);
  431. if (check_next1(ls, '=')) return TK_GE; /* '>=' */
  432. else if (check_next1(ls, '>')) return TK_SHR; /* '>>' */
  433. else return '>';
  434. }
  435. case '/': {
  436. next(ls);
  437. if (check_next1(ls, '/')) return TK_IDIV; /* '//' */
  438. else return '/';
  439. }
  440. case '~': {
  441. next(ls);
  442. if (check_next1(ls, '=')) return TK_NE; /* '~=' */
  443. else return '~';
  444. }
  445. case ':': {
  446. next(ls);
  447. if (check_next1(ls, ':')) return TK_DBCOLON; /* '::' */
  448. else return ':';
  449. }
  450. case '"': case '\'': { /* short literal strings */
  451. read_string(ls, ls->current, seminfo);
  452. return TK_STRING;
  453. }
  454. case '.': { /* '.', '..', '...', or number */
  455. save_and_next(ls);
  456. if (check_next1(ls, '.')) {
  457. if (check_next1(ls, '.'))
  458. return TK_DOTS; /* '...' */
  459. else return TK_CONCAT; /* '..' */
  460. }
  461. else if (!lisdigit(ls->current)) return '.';
  462. else return read_numeral(ls, seminfo);
  463. }
  464. case '0': case '1': case '2': case '3': case '4':
  465. case '5': case '6': case '7': case '8': case '9': {
  466. return read_numeral(ls, seminfo);
  467. }
  468. case EOZ: {
  469. return TK_EOS;
  470. }
  471. default: {
  472. if (lislalpha(ls->current)) { /* identifier or reserved word? */
  473. TString *ts;
  474. do {
  475. save_and_next(ls);
  476. } while (lislalnum(ls->current));
  477. ts = luaX_newstring(ls, luaZ_buffer(ls->buff),
  478. luaZ_bufflen(ls->buff));
  479. seminfo->ts = ts;
  480. if (isreserved(ts)) /* reserved word? */
  481. return ts->extra - 1 + FIRST_RESERVED;
  482. else {
  483. return TK_NAME;
  484. }
  485. }
  486. else { /* single-char tokens ('+', '*', '%', '{', '}', ...) */
  487. int c = ls->current;
  488. next(ls);
  489. return c;
  490. }
  491. }
  492. }
  493. }
  494. }
  495. void luaX_next (LexState *ls) {
  496. ls->lastline = ls->linenumber;
  497. if (ls->lookahead.token != TK_EOS) { /* is there a look-ahead token? */
  498. ls->t = ls->lookahead; /* use this one */
  499. ls->lookahead.token = TK_EOS; /* and discharge it */
  500. }
  501. else
  502. ls->t.token = llex(ls, &ls->t.seminfo); /* read next token */
  503. }
  504. int luaX_lookahead (LexState *ls) {
  505. lua_assert(ls->lookahead.token == TK_EOS);
  506. ls->lookahead.token = llex(ls, &ls->lookahead.seminfo);
  507. return ls->lookahead.token;
  508. }