1 /* NOTE: this is derived from Henry Spencer's regexp code, and should not
2 * confused with the original package (see point 3 below). Thanks, Henry!
5 /* Additional note: this code is very heavily munged from Henry's version
6 * in places. In some spots I've traded clarity for efficiency, so don't
7 * blame Henry for some of the lack of readability.
10 /* $RCSfile: regcomp.c,v $$Revision: 4.0.1.3 $$Date: 91/11/05 18:22:28 $
13 * Revision 4.0.1.3 91/11/05 18:22:28 lwall
14 * patch11: minimum match length calculation in regexp is now cumulative
15 * patch11: initial .* in pattern had dependency on value of $*
16 * patch11: certain patterns made use of garbage pointers from uncleared memory
17 * patch11: prepared for ctype implementations that don't define isascii()
19 * Revision 4.0.1.2 91/06/07 11:48:24 lwall
20 * patch4: new copyright notice
21 * patch4: /(x+) \1/ incorrectly optimized to not match "xxx xx"
22 * patch4: // wouldn't use previous pattern if it started with a null character
24 * Revision 4.0.1.1 91/04/12 09:04:45 lwall
25 * patch1: random cleanup in cpp namespace
27 * Revision 4.0 91/03/20 01:39:01 lwall
33 * regcomp and regexec -- regsub and regerror are not used in perl
35 * Copyright (c) 1986 by University of Toronto.
36 * Written by Henry Spencer. Not derived from licensed software.
38 * Permission is granted to anyone to use this software for any
39 * purpose on any computer system, and to redistribute it freely,
40 * subject to the following restrictions:
42 * 1. The author is not responsible for the consequences of use of
43 * this software, no matter how awful, even if they arise
46 * 2. The origin of this software must not be misrepresented, either
47 * by explicit claim or by omission.
49 * 3. Altered versions must be plainly marked as such, and must not
50 * be misrepresented as being the original software.
53 **** Alterations to Henry's code are...
55 **** Copyright (c) 1991, Larry Wall
57 **** You may distribute under the terms of either the GNU General Public
58 **** License or the Artistic License, as specified in the README file.
61 * Beware that some of this code is subtly aware of the way operator
62 * precedence is structured in regular expressions. Serious changes in
63 * regular-expression syntax might require a total rethink.
71 # if defined(BUGGY_MSC6)
72 /* MSC 6.00A breaks on op/regexp.t test 85 unless we turn this off */
73 # pragma optimize("a",off)
74 /* But MSC 6.00A is happy with 'w', for aliases only across function calls*/
75 # pragma optimize("w",on )
76 # endif /* BUGGY_MSC6 */
83 #define ISMULT1(c) ((c) == '*' || (c) == '+' || (c) == '?')
84 #define ISMULT2(s) ((*s) == '*' || (*s) == '+' || (*s) == '?' || \
85 ((*s) == '{' && regcurly(s)))
86 #define META "^$.[()|?+*\\"
89 #undef SPSTART /* dratted cpp namespace... */
92 * Flags to be passed up and down.
94 #define HASWIDTH 01 /* Known never to match null string. */
95 #define SIMPLE 02 /* Simple enough to be STAR/PLUS operand. */
96 #define SPSTART 04 /* Starts with * or +. */
97 #define WORST 0 /* Worst case. */
100 * Global work variables for regcomp().
102 static char *regprecomp; /* uncompiled string. */
103 static char *regparse; /* Input-scan pointer. */
104 static char *regxend; /* End of input for compile */
105 static int regnpar; /* () count. */
106 static char *regcode; /* Code-emit pointer; ®dummy = don't. */
107 static long regsize; /* Code size. */
109 static int regsawbracket; /* Did we do {d,d} trick? */
110 static int regsawback; /* Did we see \1, ...? */
113 * Forward declarations for regcomp()'s friends.
115 STATIC int regcurly();
117 STATIC char *regbranch();
118 STATIC char *regpiece();
119 STATIC char *regatom();
120 STATIC char *regclass();
121 STATIC char *regnode();
122 STATIC char *reganode();
124 STATIC void reginsert();
125 STATIC void regtail();
126 STATIC void regoptail();
129 - regcomp - compile a regular expression into internal code
131 * We can't allocate space until we know how big the compiled form will be,
132 * but we can't compile it (and thus know how big it is) until we've got a
133 * place to put the code. So we cheat: we compile it twice, once with code
134 * generation turned off and size counting turned on, and once "for real".
135 * This also means that we don't allocate space until we are sure that the
136 * thing really will compile successfully, and we never have to move the
137 * code and thus invalidate pointers into it. (Note that it has to be in
138 * one piece because free() must be able to free it all.) [NB: not true in perl]
140 * Beware that the optimization-preparation code in here knows about some
141 * of the structure of the compiled regexp. [I'll say.]
144 regcomp(exp,xend,fold)
151 register STR *longish;
154 register char *first;
160 extern char *safemalloc();
161 extern char *savestr();
166 fatal("NULL regexp argument");
168 /* First pass: determine size, legality. */
172 regprecomp = nsavestr(exp,xend-exp);
179 if (reg(0, &flags) == NULL) {
180 Safefree(regprecomp);
185 /* Small enough for pointer-storage convention? */
186 if (regsize >= 32767L) /* Probably could be 65535L. */
187 FAIL("regexp too big");
189 /* Allocate space. */
190 Newc(1001, r, sizeof(regexp) + (unsigned)regsize, char, regexp);
192 FAIL("regexp out of space");
194 /* Second pass: emit code. */
196 bcopy(regprecomp,exp,xend-exp);
197 r->prelen = xend-exp;
198 r->precomp = regprecomp;
199 r->subbeg = r->subbase = NULL;
202 regcode = r->program;
204 if (reg(0, &flags) == NULL)
207 /* Dig out information for optimizations. */
208 r->regstart = Nullstr; /* Worst-case defaults. */
210 r->regmust = Nullstr;
212 r->regstclass = Nullch;
213 scan = r->program+1; /* First BRANCH. */
214 if (OP(regnext(scan)) == END) {/* Only one top-level choice. */
215 scan = NEXTOPER(scan);
218 while ((OP(first) == OPEN && (sawopen = 1)) ||
219 (OP(first) == BRANCH && OP(regnext(first)) != BRANCH) ||
220 (OP(first) == PLUS) ||
221 (OP(first) == CURLY && ARG1(first) > 0) ) {
222 if (OP(first) == PLUS)
225 first += regarglen[OP(first)];
226 first = NEXTOPER(first);
229 /* Starting-point info. */
231 if (OP(first) == EXACTLY) {
233 str_make(OPERAND(first)+1,*OPERAND(first));
234 if (r->regstart->str_cur > !(sawstudy|fold))
235 fbmcompile(r->regstart,fold);
237 else if ((exp = index(simple,OP(first))) && exp > simple)
238 r->regstclass = first;
239 else if (OP(first) == BOUND || OP(first) == NBOUND)
240 r->regstclass = first;
241 else if (OP(first) == BOL ||
242 (OP(first) == STAR && OP(NEXTOPER(first)) == ANY) ) {
243 /* kinda turn .* into ^.* */
244 r->reganch = ROPT_ANCH | ROPT_IMPLICIT;
245 first = NEXTOPER(first);
248 if (sawplus && (!sawopen || !regsawback))
249 r->reganch |= ROPT_SKIP; /* x+ must match 1st of run */
253 fprintf(stderr,"first %d next %d offset %d\n",
254 OP(first), OP(NEXTOPER(first)), first - scan);
257 * If there's something expensive in the r.e., find the
258 * longest literal string that must appear and make it the
259 * regmust. Resolve ties in favor of later strings, since
260 * the regstart check works with the beginning of the r.e.
261 * and avoiding duplication strengthens checking. Not a
262 * strong reason, but sufficient in the absence of others.
263 * [Now we resolve ties in favor of the earlier string if
264 * it happens that curback has been invalidated, since the
265 * earlier string may buy us something the later one won't.]
267 longish = str_make("",0);
268 longest = str_make("",0);
274 while (OP(scan) != END) {
275 if (OP(scan) == BRANCH) {
276 if (OP(regnext(scan)) == BRANCH) {
278 while (OP(scan) == BRANCH)
279 scan = regnext(scan);
281 else /* single branch is ok */
282 scan = NEXTOPER(scan);
284 if (OP(scan) == EXACTLY) {
288 while (OP(t = regnext(scan)) == CLOSE)
290 minlen += *OPERAND(first);
291 if (curback - backish == len) {
292 str_ncat(longish, OPERAND(first)+1,
294 len += *OPERAND(first);
295 curback += *OPERAND(first);
296 first = regnext(scan);
298 else if (*OPERAND(first) >= len + (curback >= 0)) {
299 len = *OPERAND(first);
300 str_nset(longish, OPERAND(first)+1,len);
303 first = regnext(scan);
306 curback += *OPERAND(first);
308 else if (index(varies,OP(scan))) {
311 if (longish->str_cur > longest->str_cur) {
312 str_sset(longest,longish);
315 str_nset(longish,"",0);
316 if (OP(scan) == PLUS &&
317 index(simple,OP(NEXTOPER(scan))))
319 else if (OP(scan) == CURLY &&
320 index(simple,OP(NEXTOPER(scan)+4)))
321 minlen += ARG1(scan);
323 else if (index(simple,OP(scan))) {
327 if (longish->str_cur > longest->str_cur) {
328 str_sset(longest,longish);
331 str_nset(longish,"",0);
333 scan = regnext(scan);
336 /* Prefer earlier on tie, unless we can tail match latter */
338 if (longish->str_cur + (OP(first) == EOL) > longest->str_cur) {
339 str_sset(longest,longish);
343 str_nset(longish,"",0);
348 !fbminstr((unsigned char*) r->regstart->str_ptr,
349 (unsigned char *) r->regstart->str_ptr
350 + r->regstart->str_cur,
355 r->regmust = longest;
358 r->regback = backest;
360 > !(sawstudy || fold || OP(first) == EOL) )
361 fbmcompile(r->regmust,fold);
362 r->regmust->str_u.str_useful = 100;
363 if (OP(first) == EOL && longish->str_cur)
364 r->regmust->str_pok |= SP_TAIL;
373 r->do_folding = fold;
374 r->nparens = regnpar - 1;
376 Newz(1002, r->startp, regnpar, char*);
377 Newz(1002, r->endp, regnpar, char*);
386 - reg - regular expression, i.e. main body or parenthesized thing
388 * Caller must absorb opening parenthesis.
390 * Combining parenthesis handling with the base level of regular expression
391 * is a trifle forced, but the need to tie the tails of the branches to what
392 * follows makes it hard to avoid.
396 int paren; /* Parenthesized? */
401 register char *ender;
405 *flagp = HASWIDTH; /* Tentatively. */
407 /* Make an OPEN node, if parenthesized. */
411 ret = reganode(OPEN, parno);
415 /* Pick up the branches, linking them together. */
416 br = regbranch(&flags);
420 regtail(ret, br); /* OPEN -> first. */
423 if (!(flags&HASWIDTH))
425 *flagp |= flags&SPSTART;
426 while (*regparse == '|') {
428 br = regbranch(&flags);
431 regtail(ret, br); /* BRANCH -> BRANCH. */
432 if (!(flags&HASWIDTH))
434 *flagp |= flags&SPSTART;
437 /* Make a closing node, and hook it on the end. */
439 ender = reganode(CLOSE, parno);
441 ender = regnode(END);
444 /* Hook the tails of the branches to the closing node. */
445 for (br = ret; br != NULL; br = regnext(br))
446 regoptail(br, ender);
448 /* Check for proper termination. */
449 if (paren && *regparse++ != ')') {
450 FAIL("unmatched () in regexp");
451 } else if (!paren && regparse < regxend) {
452 if (*regparse == ')') {
453 FAIL("unmatched () in regexp");
455 FAIL("junk on end of regexp"); /* "Can't happen". */
463 - regbranch - one alternative of an | operator
465 * Implements the concatenation operator.
472 register char *chain;
473 register char *latest;
476 *flagp = WORST; /* Tentatively. */
478 ret = regnode(BRANCH);
480 while (regparse < regxend && *regparse != '|' && *regparse != ')') {
481 latest = regpiece(&flags);
484 *flagp |= flags&HASWIDTH;
485 if (chain == NULL) /* First piece. */
486 *flagp |= flags&SPSTART;
488 regtail(chain, latest);
491 if (chain == NULL) /* Loop ran zero times. */
492 (void) regnode(NOTHING);
498 - regpiece - something followed by possible [*+?]
500 * Note that the branching code sequences used for ? and the general cases
501 * of * and + are somewhat optimized: they use the same NOTHING node as
502 * both the endmarker for their branch list and the body of the last branch.
503 * It might seem that this node could be dispensed with entirely, but the
504 * endmarker role is not redundant.
514 char *origparse = regparse;
515 int orignpar = regnpar;
520 ret = regatom(&flags);
526 /* Here's a total kludge: if after the atom there's a {\d+,?\d*}
527 * then we decrement the first number by one and reset our
528 * parsing back to the beginning of the same atom. If the first number
529 * is down to 0, decrement the second number instead and fake up
530 * a ? after it. Given the way this compiler doesn't keep track
531 * of offsets on the first pass, this is the only way to replicate
532 * a piece of code. Sigh.
534 if (op == '{' && regcurly(regparse)) {
537 while (isDIGIT(*next) || *next == ',') {
546 if (*next == '}') { /* got one */
550 iter = atoi(regparse);
551 if (flags&SIMPLE) { /* we can do it right after all */
554 reginsert(CURLY, ret);
556 *flagp = (WORST|HASWIDTH);
562 if (tmp && tmp < iter)
563 fatal("Can't do {n,m} with n > m");
564 if (regcode != ®dummy) {
566 *(unsigned short *)(ret+3) = iter;
567 *(unsigned short *)(ret+5) = tmp;
569 ret[3] = iter >> 8; ret[4] = iter & 0377;
570 ret[5] = tmp >> 8; ret[6] = tmp & 0377;
576 regsawbracket++; /* remember we clobbered exp */
579 sprintf(regparse,"%.*d", max-regparse, iter - 1);
581 if (*max == ',' && max[1] != '}') {
582 if (atoi(max+1) <= 0)
583 fatal("Can't do {n,m} with n > m");
585 sprintf(max+1,"%.*d", next-(max+1), atoi(max+1) - 1);
588 if (iter != 1 || *max == ',') {
589 regparse = origparse; /* back up input pointer */
590 regnpar = orignpar; /* don't make more parens */
602 if (max == next) { /* any number more? */
604 op = '*'; /* fake up one with a star */
607 op = '?'; /* fake up optional atom */
609 sprintf(max,"%.*d", next-max, iter - 1);
614 regparse = origparse - 1; /* offset ++ below */
619 fatal("Can't do {n,0}");
622 fatal("Can't do {0}");
631 if (!(flags&HASWIDTH) && op != '?')
632 FAIL("regexp *+ operand could be empty");
633 *flagp = (op != '+') ? (WORST|SPSTART) : (WORST|HASWIDTH);
635 if (op == '*' && (flags&SIMPLE))
636 reginsert(STAR, ret);
637 else if (op == '*') {
638 /* Emit x* as (x&|), where & means "self". */
639 reginsert(BRANCH, ret); /* Either x */
640 regoptail(ret, regnode(BACK)); /* and loop */
641 regoptail(ret, ret); /* back */
642 regtail(ret, regnode(BRANCH)); /* or */
643 regtail(ret, regnode(NOTHING)); /* null. */
644 } else if (op == '+' && (flags&SIMPLE))
645 reginsert(PLUS, ret);
646 else if (op == '+') {
647 /* Emit x+ as x(&|), where & means "self". */
648 next = regnode(BRANCH); /* Either */
650 regtail(regnode(BACK), ret); /* loop back */
651 regtail(next, regnode(BRANCH)); /* or */
652 regtail(ret, regnode(NOTHING)); /* null. */
653 } else if (op == '?') {
654 /* Emit x? as (x|) */
655 reginsert(BRANCH, ret); /* Either x */
656 regtail(ret, regnode(BRANCH)); /* or */
657 next = regnode(NOTHING); /* null. */
659 regoptail(ret, next);
663 if (ISMULT2(regparse))
664 FAIL("nested *?+ in regexp");
670 - regatom - the lowest level
672 * Optimization: gobbles an entire sequence of ordinary characters so that
673 * it can turn them into a single node, which is smaller to store and
674 * faster to run. Backslashed characters are exceptions, each becoming a
675 * separate node; the code is simpler that way and it's not worth fixing.
677 * [Yes, it is worth fixing, some scripts can run twice the speed.]
686 *flagp = WORST; /* Tentatively. */
688 switch (*regparse++) {
697 *flagp |= HASWIDTH|SIMPLE;
701 *flagp |= HASWIDTH|SIMPLE;
704 ret = reg(1, &flags);
707 *flagp |= flags&(HASWIDTH|SPSTART);
711 FAIL("internal urp in regexp"); /* Supposed to be caught earlier. */
716 FAIL("?+* follows nothing in regexp");
721 ret = regnode(ALNUM);
722 *flagp |= HASWIDTH|SIMPLE;
726 ret = regnode(NALNUM);
727 *flagp |= HASWIDTH|SIMPLE;
731 ret = regnode(BOUND);
736 ret = regnode(NBOUND);
741 ret = regnode(SPACE);
742 *flagp |= HASWIDTH|SIMPLE;
746 ret = regnode(NSPACE);
747 *flagp |= HASWIDTH|SIMPLE;
751 ret = regnode(DIGIT);
752 *flagp |= HASWIDTH|SIMPLE;
756 ret = regnode(NDIGIT);
757 *flagp |= HASWIDTH|SIMPLE;
770 case '1': case '2': case '3': case '4':
771 case '5': case '6': case '7': case '8': case '9':
773 int num = atoi(regparse);
775 if (num > 9 && num >= regnpar)
779 ret = reganode(REF, num);
780 while (isDIGIT(*regparse))
787 if (regparse >= regxend)
788 FAIL("trailing \\ in regexp");
802 ret = regnode(EXACTLY);
803 regc(0); /* save spot for len */
804 for (len=0, p=regparse-1;
805 len < 127 && p < regxend;
855 ender = scanhex(++p, 2, &numlen);
862 ender = toupper(ender);
865 case '0': case '1': case '2': case '3':case '4':
866 case '5': case '6': case '7': case '8':case '9':
868 (isDIGIT(p[1]) && atoi(p) >= regnpar) ) {
869 ender = scanoct(p, 3, &numlen);
879 FAIL("trailing \\ in regexp");
890 if (regfold && isUPPER(ender))
891 ender = tolower(ender);
892 if (ISMULT2(p)) { /* Back off on ?+*. */
906 FAIL("internal disaster in regexp");
910 if (regcode != ®dummy)
926 if (regcode == ®dummy)
930 bits[c >> 3] &= ~(1 << (c & 7));
932 bits[c >> 3] |= (1 << (c & 7));
940 register int lastclass;
941 register int range = 0;
946 ret = regnode(ANYOF);
947 if (*regparse == '^') { /* Complement of range. */
954 for (class = 0; class < 32; class++)
956 if (*regparse == ']' || *regparse == '-')
957 goto skipcond; /* allow 1st char to be ] or - */
958 while (regparse < regxend && *regparse != ']') {
960 class = UCHARAT(regparse++);
962 class = UCHARAT(regparse++);
965 for (class = 'a'; class <= 'z'; class++)
966 regset(bits,def,class);
967 for (class = 'A'; class <= 'Z'; class++)
968 regset(bits,def,class);
969 for (class = '0'; class <= '9'; class++)
970 regset(bits,def,class);
971 regset(bits,def,'_');
975 regset(bits,def,' ');
976 regset(bits,def,'\t');
977 regset(bits,def,'\r');
978 regset(bits,def,'\f');
979 regset(bits,def,'\n');
983 for (class = '0'; class <= '9'; class++)
984 regset(bits,def,class);
1009 class = scanhex(regparse, 2, &numlen);
1013 class = *regparse++;
1015 class = toupper(class);
1018 case '0': case '1': case '2': case '3': case '4':
1019 case '5': case '6': case '7': case '8': case '9':
1020 class = scanoct(--regparse, 3, &numlen);
1026 if (lastclass > class)
1027 FAIL("invalid [] range in regexp");
1032 if (*regparse == '-' && regparse+1 < regxend &&
1033 regparse[1] != ']') {
1036 continue; /* do it next time */
1039 for ( ; lastclass <= class; lastclass++) {
1040 regset(bits,def,lastclass);
1041 if (regfold && isUPPER(lastclass))
1042 regset(bits,def,tolower(lastclass));
1046 if (*regparse != ']')
1047 FAIL("unmatched [] in regexp");
1053 - regnode - emit a node
1055 static char * /* Location. */
1063 if (ret == ®dummy) {
1074 if (!((long)ret & 1))
1080 *ptr++ = '\0'; /* Null "next" pointer. */
1088 - reganode - emit a node with an argument
1090 static char * /* Location. */
1099 if (ret == ®dummy) {
1110 if (!((long)ret & 1))
1116 *ptr++ = '\0'; /* Null "next" pointer. */
1119 *(unsigned short *)(ret+3) = arg;
1121 ret[3] = arg >> 8; ret[4] = arg & 0377;
1130 - regc - emit (if appropriate) a byte of code
1136 if (regcode != ®dummy)
1143 - reginsert - insert an operator in front of already-emitted operand
1145 * Means relocating the operand.
1154 register char *place;
1155 register offset = (op == CURLY ? 4 : 0);
1157 if (regcode == ®dummy) {
1159 regsize += 4 + offset;
1161 regsize += 3 + offset;
1168 regcode += 4 + offset;
1170 regcode += 3 + offset;
1176 place = opnd; /* Op node, where operand used to be. */
1180 while (offset-- > 0)
1185 - regtail - set the next-pointer at the end of a node chain
1192 register char *scan;
1193 register char *temp;
1194 register int offset;
1199 /* Find last node. */
1202 temp = regnext(scan);
1209 offset = val - scan;
1211 *(short*)(scan+1) = offset;
1216 if (OP(scan) == BACK)
1217 offset = scan - val;
1219 offset = val - scan;
1220 *(scan+1) = (offset>>8)&0377;
1221 *(scan+2) = offset&0377;
1226 - regoptail - regtail on operand of first argument; nop if operandless
1233 /* "Operandless" and "op != BRANCH" are synonymous in practice. */
1234 if (p == NULL || p == ®dummy || OP(p) != BRANCH)
1236 regtail(NEXTOPER(p), val);
1240 - regcurly - a little FSA that accepts {\d+,?\d*}
1264 - regdump - dump a regexp onto stderr in vaguely comprehensible form
1271 register char op = EXACTLY; /* Arbitrary non-END op. */
1272 register char *next;
1276 while (op != END) { /* While that wasn't END last time... */
1282 fprintf(stderr,"%2d%s", s-r->program, regprop(s)); /* Where, what. */
1285 if (next == NULL) /* Next ptr. */
1286 fprintf(stderr,"(0)");
1288 fprintf(stderr,"(%d)", (s-r->program)+(next-s));
1293 if (op == EXACTLY) {
1294 /* Literal string, where present. */
1296 while (*s != '\0') {
1302 (void)putchar('\n');
1305 /* Header fields of interest. */
1307 fprintf(stderr,"start `%s' ", r->regstart->str_ptr);
1309 fprintf(stderr,"stclass `%s' ", regprop(r->regstclass));
1310 if (r->reganch & ROPT_ANCH)
1311 fprintf(stderr,"anchored ");
1312 if (r->reganch & ROPT_SKIP)
1313 fprintf(stderr,"plus ");
1314 if (r->reganch & ROPT_IMPLICIT)
1315 fprintf(stderr,"implicit ");
1316 if (r->regmust != NULL)
1317 fprintf(stderr,"must have \"%s\" back %d ", r->regmust->str_ptr,
1319 fprintf(stderr, "minlen %d ", r->minlen);
1320 fprintf(stderr,"\n");
1324 - regprop - printable representation of opcode
1332 (void) strcpy(buf, ":");
1387 (void)sprintf(buf+strlen(buf), "CURLY {%d,%d}",
1392 (void)sprintf(buf+strlen(buf), "REF%d", ARG1(op));
1396 (void)sprintf(buf+strlen(buf), "OPEN%d", ARG1(op));
1400 (void)sprintf(buf+strlen(buf), "CLOSE%d", ARG1(op));
1410 FAIL("corrupted regexp opcode");
1413 (void) strcat(buf, p);
1416 #endif /* DEBUGGING */
1422 Safefree(r->precomp);
1423 r->precomp = Nullch;
1426 Safefree(r->subbase);
1427 r->subbase = Nullch;
1430 str_free(r->regmust);
1431 r->regmust = Nullstr;
1434 str_free(r->regstart);
1435 r->regstart = Nullstr;
1437 Safefree(r->startp);