[p5sagit/p5-mst-13.2.git] / regen_perly.pl

#!/usr/bin/perl
#
# regen_perly.pl, DAPM 12-Feb-04
#
# Copyright (c) 2004, 2005 Larry Wall
#
# Given an input file perly.y, run bison on it and produce
# the following output files:
#
# perly.h	standard bison header file with minor doctoring of
#		#line directives plus adding a #ifdef PERL_CORE
#
# perly.tab	the parser table C definitions extracted from the bison output
#		plus an extra table generated by this script.
#
# perly.act	the action case statements extracted from the bison output
#
# Note that perly.c is *not* regenerated - this is now a static file which
# is not dependent on perly.y any more.
#
# If a filename of the form foo.y is given on the command line, then
# this is used instead as the basename for all the files mentioned
# above.
#
# Note that temporary files of the form perlytmp.h and perlytmp.c are
# created and then deleted during this process
#
# Note also that this script is intended to be run on a UNIX system;
# it may work elsewhere but no specific attempt has been made to make it
# portable.

sub usage { die "usage: $0 [ -b bison_executable ] [ file.y ]\n" }

use warnings;
use strict;

my $bison = 'bison';

if (@ARGV >= 2 and $ARGV[0] eq '-b') {
    shift;
    $bison = shift;
}

my $y_file = shift || 'perly.y';

usage unless @ARGV==0 && $y_file =~ /\.y$/;

(my $h_file    = $y_file) =~ s/\.y$/.h/;
(my $act_file  = $y_file) =~ s/\.y$/.act/;
(my $tab_file  = $y_file) =~ s/\.y$/.tab/;
(my $tmpc_file = $y_file) =~ s/\.y$/tmp.c/;
(my $tmph_file = $y_file) =~ s/\.y$/tmp.h/;

# the yytranslate[] table generated by bison is ASCII/EBCDIC sensitive

die "$0: must be run on an ASCII system\n" unless ord 'A' == 65;

# check for correct version number. The constraints are:
#  * must be >= 1.24 to avoid licensing issues.
#  * it must generate the yystos[] table. Version 1.28 doesn't generate
#    this; 1.35+ does
#  * Must produce output which is extractable by the regexes below
#  * Must produce the right values.
# These last two contstraints  may well be met by earlier versions, but
# I simply haven't tested them yet. If it works for you, then modify
# the test below to allow that version too. DAPM Feb 04.

my $version = `$bison -V`;
unless ($version =~ /\b(1\.875[a-z]?|2\.[013])\b/) { die <<EOF; }

You have the wrong version of bison in your path; currently 1.875
2.0, 2.1 or 2.3 is required.  Try installing
    http://ftp.gnu.org/gnu/bison/bison-2.1.tar.gz
or similar.  Your bison identifies itself as:

$version
EOF

# creates $tmpc_file and $tmph_file
my_system("$bison -d -o $tmpc_file $y_file");

open CTMPFILE, $tmpc_file or die "Can't open $tmpc_file: $!\n";
my $clines;
{ local $/; $clines = <CTMPFILE>; }
die "failed to read $tmpc_file: length mismatch\n"
    unless length $clines == -s $tmpc_file;
close CTMPFILE;

my ($actlines, $tablines) = extract($clines);

$tablines .= make_type_tab($y_file, $tablines);

chmod 0644, $act_file;
open ACTFILE, ">$act_file" or die "can't open $act_file: $!\n";
print ACTFILE $actlines;
close ACTFILE;
chmod 0444, $act_file;

chmod 0644, $tab_file;
open TABFILE, ">$tab_file" or die "can't open $tab_file: $!\n";
print TABFILE $tablines;
close TABFILE;
chmod 0444, $tab_file;

unlink $tmpc_file;

# Wrap PERL_CORE round the symbol definitions. Also,  the
# C<#line 30 "perly.y"> confuses the Win32 resource compiler and the
# C<#line 188 "perlytmp.h"> gets picked up by make depend, so remove them.

open TMPH_FILE, $tmph_file or die "Can't open $tmph_file: $!\n";
chmod 0644, $h_file;
open H_FILE, ">$h_file" or die "Can't open $h_file: $!\n";
my $endcore_done = 0;
while (<TMPH_FILE>) {
    print H_FILE "#ifdef PERL_CORE\n" if $. == 1;
    if (!$endcore_done and /YYSTYPE_IS_DECLARED/) {
	print H_FILE "#endif /* PERL_CORE */\n";
	$endcore_done = 1;
    }
    next if /^#line \d+ ".*"/;
    print H_FILE $_;
}
close TMPH_FILE;
close H_FILE;
chmod 0444, $h_file;
unlink $tmph_file;

print "rebuilt:  $h_file $tab_file $act_file\n";

exit 0;


sub extract {
    my $clines = shift;
    my $tablines;
    my $actlines;

    $clines =~ m@
	(?:
	    ^/* YYFINAL[^\n]+\n		#optional comment
	)?
	\# \s* define \s* YYFINAL	# first #define
	.*?				# other defines + most tables
	yystos\[\]\s*=			# start of last table
	.*?
	}\s*;				# end of last table
    @xms
	or die "Can't extract tables from $tmpc_file\n";
    $tablines = $&;


    $clines =~ m@
	switch \s* \( \s* \w+ \s* \) \s* { \s*
	(
	    case \s* \d+ \s* : \s*
	    \#line [^\n]+"\Q$y_file\E"
	    .*?
	)
	}
	\s*
	( \s* /\* .*? \*/ \s* )*	# optional C-comments
	\s*
	(
	    \#line[^\n]+\.c"
	|
	    \#line[^\n]+\.simple"
	|
	    YY_SYMBOL_PRINT
	)
    @xms
	or die "Can't extract actions from $tmpc_file\n";
    $actlines = $1;

    # C<#line 188 "perlytmp.c"> gets picked up by make depend, so remove them.
    $actlines =~ s/^#line \d+ "\Q$tmpc_file\E".*$//gm;

    # convert yyvsp[nnn] into ps[nnn].val

    $actlines =~ s/yyvsp\[(.*?)\]/ps[$1].val/g
	or die "Can't convert value stack name\n";

    return $actlines. "\n", $tablines. "\n";
}

# Generate a table, yy_type_tab[], that specifies for each token, what
# type of value it holds.
#
# Read the .y file and extract a list of all the token names and
# non-terminal names; then scan the string $tablines for the table yytname,
# which gives the token index of each token/non-terminal; then use this to
# create yy_type_tab.
#
# ie given (in perly.y),
#
#   %token <opval> A
#   %token <ival>  B
#   %type  <pval>  C
#   %type  <opval> D
#
# and (in $tablines),
#
#   yytname[] = { "A" "B", "C", "D", "E" };
#
# then return
#
#    typedef enum { toketype_ival, toketype_opval, toketype_pval } toketypes;
#
#    static const toketypes yy_type_tab[]
#          = { toketype_opval, toketype_ival, toketype_pval,
#                toketype_opval, toketype_ival }
#
# where "E" has the default type. The default type is determined
# by the __DEFAULT__ comment  next to the appropriate union member in
# perly.y

sub make_type_tab {
    my ($y_file, $tablines) = @_;
    my %tokens;
    my %types;
    my $default_token;
    open my $fh, '<', $y_file or die "Can't open $y_file: $!\n";
    while (<$fh>) {
	if (/(\$\d+)\s*=/) {
	    warn "$y_file:$.: dangerous assignment to $1: $_";
	}

	if (/__DEFAULT__/) {
	    m{(\w+) \s* ; \s* /\* \s* __DEFAULT__}x
		or die "$y_file: can't parse __DEFAULT__ line: $_";
	    die "$y_file: duplicate __DEFAULT__ line: $_"
		    if defined $default_token;
	    $default_token = $1;
	    next;
	}

	next unless /^%(token|type)/;
	s/^%(token|type)\s+<(\w+)>\s+//
	    or die "$y_file: unparseable token/type line: $_";
	$tokens{$_} = $2 for (split ' ', $_);
	$types{$2} = 1;
    }
    die "$y_file: no __DEFAULT__ token defined\n" unless $default_token;
    $types{$default_token} = 1;

    $tablines =~ /^\Qstatic const char *const yytname[] =\E\n
	    {\n
	    (.*?)
	    ^};
	    /xsm
	or die "Can't extract yytname[] from table string\n";
    my $fields = $1;
    $fields =~ s{"([^"]+)"}
		{ "toketype_" .
		    (defined $tokens{$1} ? $tokens{$1} : $default_token)
		}ge;
    $fields =~ s/, \s* 0 \s* $//x
	or die "make_type_tab: couldn't delete trailing ',0'\n";

    return 
	  "\ntypedef enum {\n\t"
	. join(", ", map "toketype_$_", sort keys %types)
	. "\n} toketypes;\n\n"
	. "/* type of each token/terminal */\n"
	. "static const toketypes yy_type_tab[] =\n{\n"
	. $fields
	. "\n};\n";
}


sub my_system {
    system(@_);
    if ($? == -1) {
	die "failed to execute command '@_': $!\n";
    }
    elsif ($? & 127) {
	die sprintf "command '@_' died with signal %d\n",
	    ($? & 127);
    }
    elsif ($? >> 8) {
	die sprintf "command '@_' exited with value %d\n", $? >> 8;
    }
}
Commit	Line	Data
0de566d7	1	#!/usr/bin/perl
	2	#
	3	# regen_perly.pl, DAPM 12-Feb-04
	4	#
1d325971	5	# Copyright (c) 2004, 2005 Larry Wall
0de566d7	6	#
	7	# Given an input file perly.y, run bison on it and produce
	8	# the following output files:
	9	#
	10	# perly.h standard bison header file with minor doctoring of
	11	# #line directives plus adding a #ifdef PERL_CORE
	12	#
	13	# perly.tab the parser table C definitions extracted from the bison output
0539ab63	14	# plus an extra table generated by this script.
0de566d7	15	#
	16	# perly.act the action case statements extracted from the bison output
	17	#
	18	# Note that perly.c is not regenerated - this is now a static file which
	19	# is not dependent on perly.y any more.
	20	#
	21	# If a filename of the form foo.y is given on the command line, then
	22	# this is used instead as the basename for all the files mentioned
	23	# above.
	24	#
	25	# Note that temporary files of the form perlytmp.h and perlytmp.c are
	26	# created and then deleted during this process
	27	#
	28	# Note also that this script is intended to be run on a UNIX system;
	29	# it may work elsewhere but no specific attempt has been made to make it
	30	# portable.
	31
	32	sub usage { die "usage: $0 [ -b bison_executable ] [ file.y ]\n" }
	33
	34	use warnings;
	35	use strict;
	36
	37	my $bison = 'bison';
	38
	39	if (@ARGV >= 2 and $ARGV[0] eq '-b') {
	40	shift;
	41	$bison = shift;
	42	}
	43
	44	my $y_file = shift \|\| 'perly.y';
	45
	46	usage unless @ARGV==0 && $y_file =~ /\.y$/;
	47
	48	(my $h_file = $y_file) =~ s/\.y$/.h/;
	49	(my $act_file = $y_file) =~ s/\.y$/.act/;
	50	(my $tab_file = $y_file) =~ s/\.y$/.tab/;
	51	(my $tmpc_file = $y_file) =~ s/\.y$/tmp.c/;
	52	(my $tmph_file = $y_file) =~ s/\.y$/tmp.h/;
	53
	54	# the yytranslate[] table generated by bison is ASCII/EBCDIC sensitive
	55
	56	die "$0: must be run on an ASCII system\n" unless ord 'A' == 65;
	57
	58	# check for correct version number. The constraints are:
	59	# * must be >= 1.24 to avoid licensing issues.
	60	# * it must generate the yystos[] table. Version 1.28 doesn't generate
	61	# this; 1.35+ does
	62	# * Must produce output which is extractable by the regexes below
	63	# * Must produce the right values.
	64	# These last two contstraints may well be met by earlier versions, but
	65	# I simply haven't tested them yet. If it works for you, then modify
	66	# the test below to allow that version too. DAPM Feb 04.
	67
	68	my $version = `$bison -V`;
3797f23d	69	unless ($version =~ /\b(1\.875[a-z]?\|2\.[013])\b/) { die <<EOF; }
0de566d7	70
2080282f	71	You have the wrong version of bison in your path; currently 1.875
3797f23d	72	2.0, 2.1 or 2.3 is required. Try installing
20515881	73	http://ftp.gnu.org/gnu/bison/bison-2.1.tar.gz
0de566d7	74	or similar. Your bison identifies itself as:
	75
	76	$version
	77	EOF
	78
	79	# creates $tmpc_file and $tmph_file
	80	my_system("$bison -d -o $tmpc_file $y_file");
	81
	82	open CTMPFILE, $tmpc_file or die "Can't open $tmpc_file: $!\n";
	83	my $clines;
	84	{ local $/; $clines = <CTMPFILE>; }
	85	die "failed to read $tmpc_file: length mismatch\n"
	86	unless length $clines == -s $tmpc_file;
	87	close CTMPFILE;
	88
	89	my ($actlines, $tablines) = extract($clines);
	90
d5c6462e	91	$tablines .= make_type_tab($y_file, $tablines);
0539ab63	92
0de566d7	93	chmod 0644, $act_file;
	94	open ACTFILE, ">$act_file" or die "can't open $act_file: $!\n";
	95	print ACTFILE $actlines;
	96	close ACTFILE;
	97	chmod 0444, $act_file;
	98
	99	chmod 0644, $tab_file;
	100	open TABFILE, ">$tab_file" or die "can't open $tab_file: $!\n";
	101	print TABFILE $tablines;
	102	close TABFILE;
	103	chmod 0444, $tab_file;
	104
	105	unlink $tmpc_file;
	106
	107	# Wrap PERL_CORE round the symbol definitions. Also, the
96f4e226	108	# C<#line 30 "perly.y"> confuses the Win32 resource compiler and the
96f4e226	109	# C<#line 188 "perlytmp.h"> gets picked up by make depend, so remove them.
0de566d7	110
	111	open TMPH_FILE, $tmph_file or die "Can't open $tmph_file: $!\n";
	112	chmod 0644, $h_file;
	113	open H_FILE, ">$h_file" or die "Can't open $h_file: $!\n";
	114	my $endcore_done = 0;
	115	while (<TMPH_FILE>) {
	116	print H_FILE "#ifdef PERL_CORE\n" if $. == 1;
	117	if (!$endcore_done and /YYSTYPE_IS_DECLARED/) {
	118	print H_FILE "#endif /* PERL_CORE */\n";
	119	$endcore_done = 1;
	120	}
96f4e226	121	next if /^#line \d+ ".*"/;
0de566d7	122	print H_FILE $_;
	123	}
	124	close TMPH_FILE;
	125	close H_FILE;
	126	chmod 0444, $h_file;
	127	unlink $tmph_file;
	128
	129	print "rebuilt: $h_file $tab_file $act_file\n";
	130
	131	exit 0;
	132
	133
	134	sub extract {
	135	my $clines = shift;
	136	my $tablines;
	137	my $actlines;
	138
	139	$clines =~ m@
	140	(?:
	141	^/* YYFINAL[^\n]+\n #optional comment
	142	)?
	143	\# \s* define \s* YYFINAL # first #define
	144	.*? # other defines + most tables
	145	yystos\[\]\s*= # start of last table
	146	.*?
	147	}\s*; # end of last table
	148	@xms
	149	or die "Can't extract tables from $tmpc_file\n";
	150	$tablines = $&;
	151
	152
	153	$clines =~ m@
	154	switch \s* \( \s* \w+ \s* \) \s* { \s*
	155	(
	156	case \s* \d+ \s* : \s*
2ade6388	157	\#line [^\n]+"\Q$y_file\E"
0de566d7	158	.*?
	159	)
	160	}
	161	\s*
	162	( \s* /\* .? \/ \s* )* # optional C-comments
	163	\s*
	164	(
	165	\#line[^\n]+\.c"
	166	\|
	167	\#line[^\n]+\.simple"
3797f23d	168	\|
3797f23d	169	YY_SYMBOL_PRINT
0de566d7	170	)
	171	@xms
	172	or die "Can't extract actions from $tmpc_file\n";
	173	$actlines = $1;
	174
0d6f9730	175	# C<#line 188 "perlytmp.c"> gets picked up by make depend, so remove them.
	176	$actlines =~ s/^#line \d+ "\Q$tmpc_file\E".*$//gm;
	177
1654d593	178	# convert yyvsp[nnn] into ps[nnn].val
	179
	180	$actlines =~ s/yyvsp\[(.*?)\]/ps[$1].val/g
	181	or die "Can't convert value stack name\n";
	182
0de566d7	183	return $actlines. "\n", $tablines. "\n";
	184	}
	185
d5c6462e	186	# Generate a table, yy_type_tab[], that specifies for each token, what
d5c6462e	187	# type of value it holds.
0539ab63	188	#
d5c6462e	189	# Read the .y file and extract a list of all the token names and
	190	# non-terminal names; then scan the string $tablines for the table yytname,
	191	# which gives the token index of each token/non-terminal; then use this to
	192	# create yy_type_tab.
0539ab63	193	#
d5c6462e	194	# ie given (in perly.y),
	195	#
	196	# %token <opval> A
	197	# %token <ival> B
	198	# %type <pval> C
	199	# %type <opval> D
	200	#
	201	# and (in $tablines),
	202	#
	203	# yytname[] = { "A" "B", "C", "D", "E" };
0539ab63	204	#
0539ab63	205	# then return
d5c6462e	206	#
	207	# typedef enum { toketype_ival, toketype_opval, toketype_pval } toketypes;
	208	#
	209	# static const toketypes yy_type_tab[]
	210	# = { toketype_opval, toketype_ival, toketype_pval,
	211	# toketype_opval, toketype_ival }
	212	#
	213	# where "E" has the default type. The default type is determined
	214	# by the __DEFAULT__ comment next to the appropriate union member in
	215	# perly.y
0539ab63	216
d5c6462e	217	sub make_type_tab {
0539ab63	218	my ($y_file, $tablines) = @_;
0539ab63	219	my %tokens;
d5c6462e	220	my %types;
d5c6462e	221	my $default_token;
0539ab63	222	open my $fh, '<', $y_file or die "Can't open $y_file: $!\n";
0539ab63	223	while (<$fh>) {
29522234	224	if (/(\$\d+)\s*=/) {
	225	warn "$y_file:$.: dangerous assignment to $1: $_";
	226	}
	227
d5c6462e	228	if (/__DEFAULT__/) {
	229	m{(\w+) \s* ; \s* /\* \s* __DEFAULT__}x
	230	or die "$y_file: can't parse __DEFAULT__ line: $_";
	231	die "$y_file: duplicate __DEFAULT__ line: $_"
	232	if defined $default_token;
	233	$default_token = $1;
	234	next;
	235	}
	236
	237	next unless /^%(token\|type)/;
	238	s/^%(token\|type)\s+<(\w+)>\s+//
	239	or die "$y_file: unparseable token/type line: $_";
	240	$tokens{$_} = $2 for (split ' ', $_);
	241	$types{$2} = 1;
0539ab63	242	}
d5c6462e	243	die "$y_file: no __DEFAULT__ token defined\n" unless $default_token;
d5c6462e	244	$types{$default_token} = 1;
0539ab63	245
	246	$tablines =~ /^\Qstatic const char *const yytname[] =\E\n
	247	{\n
	248	(.*?)
	249	^};
	250	/xsm
	251	or die "Can't extract yytname[] from table string\n";
	252	my $fields = $1;
d5c6462e	253	$fields =~ s{"([^"]+)"}
	254	{ "toketype_" .
	255	(defined $tokens{$1} ? $tokens{$1} : $default_token)
	256	}ge;
	257	$fields =~ s/, \s* 0 \s* $//x
	258	or die "make_type_tab: couldn't delete trailing ',0'\n";
	259
0539ab63	260	return
d5c6462e	261	"\ntypedef enum {\n\t"
	262	. join(", ", map "toketype_$_", sort keys %types)
	263	. "\n} toketypes;\n\n"
	264	. "/* type of each token/terminal */\n"
d5c6462e	265	. "static const toketypes yy_type_tab[] =\n{\n"
	266	. $fields
	267	. "\n};\n";
0539ab63	268	}
	269
	270
0de566d7	271	sub my_system {
	272	system(@_);
	273	if ($? == -1) {
d5c6462e	274	die "failed to execute command '@_': $!\n";
0de566d7	275	}
	276	elsif ($? & 127) {
	277	die sprintf "command '@_' died with signal %d\n",
	278	($? & 127);
	279	}
	280	elsif ($? >> 8) {
	281	die sprintf "command '@_' exited with value %d\n", $? >> 8;
	282	}
	283	}