//! LINEPROCX.CPP Version 2.97 (09/03/2022)
//. J.L. Villar
//. An unnecessary tool that "simplifies" line processing on data text files
//. It should be simpler and far less powerful than awk
#define VER_MAJOR	2
#define VER_MINOR	97
#define DATE		"09/03/2022"
//
// History:
//
// I forgot the origins of the program, but it could date to year 2000 
// or even earlier. A previous version (probably without inline templates) 
// existed under the name lineproc (without the 'x'), dating to may 22 2006.
//
// [22/05/2006] This is the last known version of lineproc 
// --------------------- Version 1.00 ----------------------------------------
// [06/07/2006] #q now inserts a double quote and #* expands to the entire 
//              data record.
//              This is the first known version of lineprocx, and probably 
//              the first one compiled on a Linux machine.
// --------------------- Version 1.10 ----------------------------------------
// [06/07/2006] Inline template is added, without template nesting.
// --------------------- Version 1.15 ----------------------------------------
// [15/06/2007] Inline templates can be nested (template stack added).
// --------------------- Version 1.20 ----------------------------------------
// [23/10/2007] Conditional expansion and named inline templates were added,
//              and the unnamed inline template syntax was altered to the
//              double bracket #[[ #]]. Also, data file insertion (with #<...>)
//              was added at that time. 
// --------------------- Version 1.22 ----------------------------------------
// [22/12/2017] Added the #+ functionality (parse next data record during
//              template expansion).
// --------------------- Version 1.24 ----------------------------------------
// [16/03/2018] Added the #.0,...,#.9 functionality (command line argument 
//              access).
// --------------------- Version 1.30 ----------------------------------------
// [19/03/2018] Command line argument access also allowed in Inline Templates.  
//              Inline templates can be multiline now!
//              Variable definition is added to inline templates (in body).
//              "Direct Inline Template" syntax is also added.
// --------------------- Version 1.32 ----------------------------------------
// [20/03/2018] Changed syntax of input file #<...> to #<..., and also the
//              filename can use macro expansion commands (variables, 
//              conditional expansion, ...).
// --------------------- Version 1.33 ----------------------------------------
// [21/03/2018] Fixed a bug with '#+' (now, it is ignored if skipping>=0).
// --------------------- Version 2.00 ----------------------------------------
// [05/10/2018] Added two additional comparison operators: postfix and superstr.
//		Added the execution of a direct template stored in a variable.
//		This implied nested template expansions, and sharing of some
//		expansion stacks (marks and skipping states).
// [06/10/2018] Tolerating an empty string as a possible direct template 
//		stored in a variable.
// --------------------- Version 2.02 ----------------------------------------
// [07/10/2018] In multiline templates, lines starting with '##' are skipped.
//		Special variables __FSEP__ and __RSEP__ store now the field and
//		(output) record separators. Thus, the use of argv[3] and argv[4]
//		is deprecated (but still supported).
// --------------------- Version 2.10 ----------------------------------------
// [04/03/2019] The previous reserved variable names are replaced by __IFSEP__
//		__OFSEP__ and __ORSEP__, where now the field separators for
//		input and output are managed separately. Thus, the template 
//		#@#*#@ replaces the input field separator by the output one.
//		Most limitations in inline templates header were removed.
//		Now, head and tail in inline templates are not only strings
//		but full subtemplates. Variables, conditional expansion,
//		operators like #+ and #n and fields #1,...,#* are allowed.
//		The only difference is that #n cannot be used to set the
//		record number in the inline template, and it expands to the
//		record number even when used in the head.
//		The command #c is only allowed in the head of the outer template.
//		But it could be enabled also in the head of an inline template.
// --------------------- Version 2.11 ----------------------------------------
// [06/03/2019] Most restrictions of the outer template are removed. Now, the
//		template head and tail are subtemplates, and conditional
//		macro expansion and variable access are enabled. 
// --------------------- Version 2.50 ----------------------------------------
// [09/03/2019] The previously existing (27/01/2005) math. evaluator is now 
//		integrated inside LINPROCX. Mathematical expressions are now
//		surrounded by #$[...]. The usual unary and binary arithmetic 
//		operators as well as the most common functions and constantes are 
//		supported. Multiple variable assignations and combined operators
//		like += are also implemented.
//		Expressions starting with a @ do not produce any output.
//		If the first character in the expressions (after the optional @)
//		is &, (e.g., #$[&name]) the expression to be evaluated is the 
//		contents of the LINEPROCX variable "name". If the starting 
//		character is '`', then the math expression is obtained by expanding
//		the variable contents.
//		The commands inside the expression #$[...] are executed before 
//		evaluating it. Thus, #$[&name] and #$[`name] are abbreviations of
//		#$[#&[name]] and #$[#`[name]]. 
// --------------------- Version 2.51 ----------------------------------------
// [11/03/2019] All stacks are made dynamic (but they only grow and never shrinks).
//		Almost all stacks can be made shrinkable, except for the unnamed 
//		and active template stacks (because items above the stack top
//		can be reused without initialization).
//		Named inline template and variable tables are still static. To 
//		make them dynamic, an update() mechanism must be provided, in order
//		to use the heap instead of the static memory. 
//		Doing that would allow larger expansion and variable content
//		buffers, that could also be located on the heap.
//		However, increasing the number of named objects in a table 
//		requires first improving the performance of the search algorithm.
//		A hash table of binary trees is recommended.
// [12/03/2019] Added dynamic memory variable names and contents, and an advanced
//		table search (hash valued + binary trees).
//		Now, there is no actual limitation in the number of variables.
// [13/03/2019] Fixed a problem with the recently introduced direct template stack.
//		Direct templates where parsed recursively (due to possible nesting
//		of variable contents execution). Instead of using a auxiliary
//		template local variable, a separate direct template stack is used,
//		to save system stack space.
//		However, empty direct templates (resulting from empty variables)
//		were buggy (as they were not initially accepted as valid).
//		Now, empty inline templates are acceptable:
//			An empty direct template produces no output.
//			The execution of an empty variable causes no output.
//			The execution of an empty inline template causes no output.
//		Empty named templates can only be defined with #[-name], since
//		an empty #[name] will just activate the template "name". But you
//		can always use #[name]#@ to clear and activate the named template.
//		Using an empty #[[, does not redefine the unnamed template. It is
//		impossible to redefine an unnamed template to be empty, but you
//		can always produce the same effect with #[[#@.
//		Undefined variables resolve to the empty string (since nobody is 
//		interested in errors related to undeclared variables). However,
//		for debugging reasons, it would be useful having the option of 
//		tracking the use of undefined variables. 
//		Now, new system vars are introduced:
//			__RREP__ tracks the record repetition (like #c)
//			__NR__ tracks the record number (like #n)
//		Thus, the use of #n and #c in the header is deprecated.
// --------------------- Version 2.60 ----------------------------------------
// [14/03/2019] The template length and number of commands limitations have been
//		removed.
//		Moreover, the named templates are managed into hash tables. Therefore,
//		there is no current limitation in the number of defined named 
//		templates.
//		The only remaining static size limitations are:
//			input filename length (512 bytes)
//			input line buffer (4096 bytes)
//			expansion buffer (65535 bytes)
//			field and record separator lengths (255 bytes)
//			outer template length (65535 bytes)
//			outer template command fields (256 items)
//			auxiliary int and double print buffers (16 and 32 bytes)
//			other limitations in the  VEXPR module (math. variables)
//		All the other structures are handled with dynamical tables and
//		buffers.
// --------------------- Version 2.61 ----------------------------------------
// [18/03/2019] Hash tables are now also used for math variables. The total 
//		number of math variables is no longer limited, and the math 
//		variable name length limit is extended to 64 characters.
// --------------------- Version 2.70 ----------------------------------------
// [24/02/2020] Math error messages are now more informative.
//		Setting __IFSEP__ to an empty string now identifies fields with
//		single characters.
//		In addition, the #- functionality is added. It rescans the fields
//		of the current line, perhaps with a different value of __IFSEP__.
//		Another special feature has been added to the record management.
//              The #|[(foo)] command replaces the inpur line contents by (foo)
//		and then rescans the field and continues normal precessing.
//		An important extension of the field specification in templates is
//		added: Now ranges of fields are written as, for example,
//			#[2:5,1,0,-3:-1]
//		The result is a concatenation of the corresponding fields separated
//		by __OFSEP__. 0 means the last field, -1 the second last one, etc.
//		Comma means range concatenation, and colon means a sequence of
//		consecutive fields.
//		Nested expressions are consistently handled, as in #[#[1]:#[2]].
// --------------------- Version 2.71 ----------------------------------------
// [25/02/2020] A readonly numeric system var is created: __NF__ is the number
//		of fields in the current input source line. Any writting attempt
//		on the variable is ignored.
//		Added to new commands:
//		  #_ breaks the expansion of the outer template, by skipping
//		all the subsequent elements. It also works in any direct template.
//		It independently breaks the head, the body or the tail.
//		  #% breaks the expansion (as in #_) but repeats it if the command
//		occurs in the body of the outer template (using it can easily 
//		produce infinite loops!). Direct templates are unaffected by this
//		command. Therefore, there is no way to loop in a direct template.
//		This command is ignored both in the head and in the tail of a
//		template.
// --------------------- Version 2.72-----------------------------------------
// [31/07/2020] A printing format specification is added to math expressions.
//		A new primitive is introduced: Conditional expansion controlled
//		by a mathematical expression, with syntax
//		  #?$[<expression>]?[<true block>]:[<false block>]
//		The true block is expanded if the expression evaluates to a 
//		nonzero value, and the false block is expanded otherwise.
//		The optional "mute character" @ and the printing format are still 
//		supported (but ignored). The expression can be an
//		assignement one. Extended syntax like #?$[&var] or #?$[`var]
//		is also allowed, with the natural meaning.
//		Comparison and C-like logical operators have been added
//		to the math evaluator.
//		As minor changes in the program, literal constants have been 
//		introduced replacing the explicit numeric values for the parser
//		states, and the function providing the help information was 
//		slightly changed (help text is now in a string array). 
// --------------------- Version 2.80-----------------------------------------
// [02/08/2020] The input file stack is replicated 10 times, and now there are
//		10 independent input channels, being channel 0 the default one.
//		A new command familly han been introduced:
//		  #<0,...,#<9 select the corresponding input channel 0,...,9.
//		  #<? checks whether the current channel is ready or exhausted
//		    it evaluates to 0 at EOF, and to 1otherwise.
//		  #<- closes the current file (in the current input channel)
//		  #<[filename] pushes filename into the stack associated to the
//		    current input channel.
//		The new readonly variable __INPUT__ contains the current channel.
// --------------------- Version 2.81-----------------------------------------
// [24/11/2020] Fixed a bug about the implementation of the command #_:
//		The variable enable was not set before the execution of a special
//		line (processspecial()). 
//		Now #_ or #% in an outer template does not affect subsequent 
//		outer direct template expansions.
//		Moreover, looping and halting is now allowed in the execution
//		of any direct trmplate. This affects to outer direct templates,
//		but also to filename in #<... or the variable content execution
//		commands, like #`[...] or #$[`...].
//		Nested loops are allowed to some extent (you can save a loop
//		into a variable, and then execute it inside another loop).
//		An example of this is:
//		  #!#![x]=[##$[@x=x0#]##@##?$[x-=1#]?[-##%#]:[o\n#]]
//		  #!#$[@y=80]#@#?$[y-=1]?[#$[@x0=floor(30*(1+sin(.2*y)))+1]#`[x]#%]:[]
//		Thus, you do no longer need the use of dummy lines in outer templates
//		to perform some loops that do not require variable data.
//		TODO The new loop and halt behavior inside direct templates could
//		affect the compatibility of some existing code with previous versions.
//		I will maintain a compilation switch to disable the new functionality.
// --------------------- Version 2.82-----------------------------------------
// [19/12/2020] Added the immediate output functionality #>[...], to help preventing
//		the excessive accumulation of data in the expansion buffer.
//		Changed the readline function behaviour: no error is produced on
//		input channel EOF.
//		Added the range functionality also for command line arguments.
//		Fixed a number of bugs related to the reallocatability of DynStack<>
//		and the fact that template objects do not have a meaningful copy
//		operator. Moreover, stack objects reallocation can occur while
//		some templates are expanding, and the existing references can become
//		invalid.
//		The wrong implementation started in version
//		2.51, and can cause core dumps if more than 20 nesting levels of
//		direct templates are used. There is still a problem that arises
//		when more than 37000 nesting levels are used.
//		The buggy code is the infinite loop:
//		#!#$[i=37391]#![a]=[##>[##$[i-=1#]\n#]##?$[i#]?[##`[a#]#]:[#]]
//		Now, I statically fixed the maximum number of nested unnamed templates
//		to 32 (I think that nobody would be interested in nesting more unnamed
//		templates instead of simply using direct templates or named ones).
//		Direct templates do not require a separate stack, and the currently
//		expanded direct template is created in the heap and locally referenced
//		by the expansion function, who owns the object. Direct templates are
//		actually "one-time" objects. 
//		TODO Find the cause of the problem and fix it.
// --------------------- Version 2.90-----------------------------------------
// [12/08/2021] Added a new comparison operator #?[<string>]/[<regexpr>] that tries
//		to match the regular expression <regexpr> with any substring of
//		<string>. Most of the functionalities of the extended regular
//		expressions in grep are implemented. The main difference is that
//		no backtracking mechanism is provided. Therefore, the string "aaa"
//		fails to match the regular expression "a*a" because the term "a*"
//		consumes all the characters and there is no additional 'a' for the
//		last term. On the contrary, grep searches the longest matching
//		using some backtracking mechanism, that in the above example, consumes
//		the maximum number of characters that still makes possible to match
//		the entire regular expression. There could be other differences
//		related to how back references are dealt with, when combined with
//		repetition operators, like in "((a*)*)c\2". Actually, in my system
//		grep enters an infinite loop when submitted to the input "ac".
//		Moreover, the new command syntax #/[<string>]/[<regexpr>] is added,
//		and it expands to the matched substring, or to the empty string if
//		no matches found. Thus, this command cannot tell apart a no-matching
//		situation from a zero-length string matching, as it occurs in the
//		example #/[any string]/[$], where the empty leading substring matches
//		the regular expression "$".
//		Notice that the two active characters '#' and ']' need to be escaped
//		in most situations, as occurs with many constructions in lineprocx.
//		The character '/' has a special meaning (explained below) as a 
//		separator. Therefore, the literal '/' needs to be scaped as '\/'.
//		The following syntax extension has been also added, and it requires
//		the replication of some strings to avoid overwriting them when
//		expanding the command.
//		The extended syntax #/[<string>]/[<regexpr>/<template>] first looks for
//		a matching of the regular expression, and then <template> is used to
//		compute the result of the expansion. The template can contain valid
//		back references. Unmatched back references are ignored. In the 
//		conditional expansion operator, everything after a '/' is simply 
//		ignored, and everything after a second '/' is also ignored in the 
//		#/ operator. (A third section is reserved for future use).
// --------------------- Version 2.91-----------------------------------------
// [21/08/2021] Fixed an issue (bug) related to the input field separator length:
//		If the input field separator length was changed, then the length of the
//		fields was computed wrongly (e.g., if the input field separator is
//		replaced by another one with one less character, then when printing
//		a field the first character of the original input field separator is
//		appended to it). This was because at field-separation implementation 
//		time there weren't variables or system variables, and nobody thought
//		that the imput field separator could be changed on the fly.
//		The fix is saving the pointers to the fields along with the field
//		lengths.
//		In addition, I have added the possibility that the command #/
//		dumps the matched back references to input fields, allowing for more
//		functionalities of the regular expression handling. #1 is the whole
//		matching string (back reference 0), #2 is back reference 1, and so on.
//		#* is the usual concatenation of input fields and the output field
//		separator. The system variable __NR__ is updated accordingly.
//		As back reference matchings are temporarily stored as heap objects, any
//		invalidation of them imply setting __NR__ to 0 (as if the empty line
//		were parsed). The syntax for that operation is
//		#/[<string>]/[<regexpr>/<templ>/|]
//		If the '|' is detected, in addition to the normal operation of both
//		commands, the input fields #1,#2,... are set with the back reference
//		matchings. Any further call to any of the two commands will recompute
//		the back references and then invalidate the association with input
//		fields. Similarly, any subsequent record update command	like #+, #-, #|
//		(actually, any call to parsefields()) will remove the association.
//		The regexpr conditional expansion operator does not implement this
//		new functionality, but calling it also breaks the association of back
//		references and input fields. 
//		A new command is added to rescan the input fields using as field 
//		the matchings with a given regular expression. The matching substrings
//		are also stored as input fields. The syntax is #:[<regexpr>]. As in
//		the regexpr conditional expansion operator, any character following
//		a '/', including itself, is ignored.
// --------------------- Version 2.95-----------------------------------------
// [30/08/2021] Changed the implementation of regular expressions to add backtracking.
//		The recursive approach (for parenthesized subexpressions) is changed
//		to a single function call driven by a status stack containing all
//		backtracking points (repetition operators and OR construction) as well
//		as all entrances to parenthesized subexpressions. The later also form
//		a linked list that is used to emulate the normal stack-based recursion,
//		while maintaining the possibility to resume teh execution context at
//		any backtracking point (because all contexts remain in the stack even
//		when the function leaves a parenthesized subexpression. The status
//		stack is flushed only at the end of the while matching process.
//		The implementation is experimental, and it is still subjected to some
//		performance improvements and bug fixing.
//		Interestingly, some buggy regular expressions like ((.)\2){2} that do
//		not work as expected in some distributions of grep, actually do well
//		here. However, the backtracking implemented here is quite lazy, and 
//		it stops as soon it finds a matching (and not necessarilly the longest
//		one). Repetition operators like .* often look for the (locally) longest
//		possible matching, but this is not the case when they are combined with
//		the OR construction, like in (a*|b)+ for a string like "aaaaaaaaab".
//		However, the regular expression (a+|b)+ does the work properly because
//		the a+ part always consumes a character, while a* never fails to match.
//		A simpler example is the regular expression |aaa, that will always
//		match the empty string (but it is not the case with |aaa).
//		The data structure supporting the backtracking mechanism required a
//		modification to the classes in STACK.H: two new methods and a new
//		exception object were added. Namely, the members
//		   int DynStack<T>::curridx()
//		   T& DynStack<T>::operator[] (int)
//		and the exception class x_badindex.
//		The number of back references that can be tracked is increased to 32
//		(another static limitation!). I don't think it is useful increasing it
//		much more, but it would be quite easy to remove this bound. Indeed,
//		a first parsing pass can detect the number of necessary back references
//		in the regular expression, and then the necessary arrays can be
//		properly allocatated.
//		By the way, the new (read only) system variable __VERSION__ has been
//		added. It is a float number, and currently it evaluates to 2.95.
// --------------------- Version 2.96-----------------------------------------
// [30/01/2022] Added a new command for direct inline direct template expansion,
//		without the need of temporarily storing it into a variable.
//		   #^[<templ>] expands <templ> as a direct template.
//		Notice that any command is interpreted before parsing <templ> as a
//		template. For instance, #^[pre#@inter#@post] will produce errors,
//		because the correct command is #^[pre##@inter##@post].
//		TODO Document the new command in the man page.
// --------------------- Version 2.97-----------------------------------------
// [09/03/2022]	Added a new experimental functionality "verbose template text".
//		Any text between #{ and #} is kept unchanged, but escape chars
//		are still interpreted. Nested #{ #} are properly supported, i.e., 
//		the inner #{ #} are untouched. Multiline inline templates work
//		the same way as before: #\n are simply ignored, but next line
//		must start with #, that will also be ignored along with any
//		subsequent sequence of whitespaces. This functionality would
//		help writting "delayed execution" code, like a template text
//		stored into a variable. Observe that this functionality does
//		not remove the need of using \\ or \\\\ to refer to a delayed
//		expansion escape char.
//		TODO Document the new command in the man page.

//	Future	TODO Document (or remove) the following side effect produced by
//		unfinished multiline templates: Set a variable with a string
//		terminated by a single #. Then execute it, like in
//		    #![_]=[##]#`[_]
//		This code sample will read next input line(s) as a continuation of
//		the unfinished direct template and execute it. You can produce a 
//		similar effect with #^[##]. Explore other funny effects like in 
//		    #![_]=[##]#`[_]#`[_]
//		TODO Use dynamic buffers also in the outer template and in the
//		input and expansion buffers.
//		TODO Analyze the strange behavior of "incomplete" heads in templates.
//		E.g., #[strange]#![x]=[#@a#@] seems to be partially acceptable!!!
//	Future 	TODO Document all new improvements: some size limitations removed,
//		math expr., side effects of expr (RPN is also supported!), ...
//	Future 	TODO Correct the error in the explanation of how #[[ and #]] work
//		with respect to the unnamed template head and tail.
//	Future 	TODO An interesting syntactic extension is making all [&name]
//		groups mean the same as [#&[name]], and the same with [`name].
//		But then it must be some means to escape these two characters!
//	Future 	TODO Make the use of #]] and #[] more consistent.
//	Future 	TODO Add an option for tracking the use of undefined variables
//	Future 	TODO Document the following side effect of named template definition:
//		You can redefine a template "name" while is is active in a nested
//		template expansion. The modification takes effect immediately
//		at all expansion levels, since there is a single instance of a 
//		named template, even when it is referenced several times in the
//		active template stack.

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
#include <ctype.h>

#include "vexpr.h"
#include "stack.h"
#include "hash.h"

//#define	NO_LOCAL_LOOPS_AND_HALTS_IN_DIRECT_TEMPLATES

//: Errors as exceptions

class x {
public:
	const char *m1;
	const char *m2;
	x(const char *p1 = 0,const char *p2 = 0) : m1(p1), m2(p2) {}
};

const int errbuflen=255;
static char errbuf[errbuflen+1];

static void error(const char *p1 = 0,const char *p2 = 0) {
        if (p2) {
                strncpy(errbuf,p2,errbuflen);
                strcpy(errbuf+errbuflen-3,"...");
                throw x(p1,errbuf);
        }
        throw x(p1);
}

static void error(const char *p1,char c) {
        if (c) {
                errbuf[0]='\'';
		errbuf[1]=c;
		errbuf[2]='\'';
                throw x(p1,errbuf);
        }
        throw x(p1);
}

//: Input stream management
//. Input filenames (along with file positions) are stored in a stack.
//. Only the active file (top of the stack) is open.
//. An EOF during reading causes the current file being closed and
//. discarded, then the previous file is reopened at the last position.
//. stdin can only appear at the bottom of the stack.
//. Global EOF occurs when the stack is empty.
//. On error, only the active file needs to be closed and the stack flushed.
//. A new file can be opened by closing the current input file and
//. storing the current file position. Then the new input file is pushed 
//. to the stack and opened.

struct vvfilepos {
	enum {vvfnamelen = 512}; // STATIC LIMITATION
	char name[vvfnamelen+1];
	fpos_t pos;
	vvfilepos() {name[0] = 0;}
};

#define	MAX_INPUT_CHANNELS	10

static int in_channel = 0;

DynStack<vvfilepos> fin_stk_array[]={
	DynStack<vvfilepos>("Input File Stack 0",10),
	DynStack<vvfilepos>("Input File Stack 1",10),
	DynStack<vvfilepos>("Input File Stack 2",10),
	DynStack<vvfilepos>("Input File Stack 3",10),
	DynStack<vvfilepos>("Input File Stack 4",10),
	DynStack<vvfilepos>("Input File Stack 5",10),
	DynStack<vvfilepos>("Input File Stack 6",10),
	DynStack<vvfilepos>("Input File Stack 7",10),
	DynStack<vvfilepos>("Input File Stack 8",10),
	DynStack<vvfilepos>("Input File Stack 9",10)
};

FILE *fin_array[MAX_INPUT_CHANNELS] = {0};

#define fin 	(fin_array[in_channel])
#define fin_stk	(fin_stk_array[in_channel])

//static FILE *fin = 0;

static void vvclose() {
	for (in_channel = 0; in_channel < MAX_INPUT_CHANNELS; in_channel++) {
		fin_stk.flush();
		if (fin && fin != stdin) {
			fflush(fin);
			fclose(fin);
			fin = 0;
		}
	}
}

static void push_file(const char *name) {
	if (!name || !*name) error("No input file name");
	if (strlen(name) > vvfilepos::vvfnamelen) error("input filename too long",name);
	if (!fin_stk.empty()) {
		if (fin && fin != stdin) {
			if (fgetpos(fin,&fin_stk.currit().pos)) error("Closing input file","fgetpos");
			fclose(fin);
		}
		fin = 0;
	}
	if (!strcmp(name,"-")) fin = stdin;
	else {
		fin = fopen(name,"rt");
		if (!fin) error("Opening input file",name);
	}
	fin_stk.push();	
	if (fin != stdin && fgetpos(fin,&fin_stk.currit().pos)) error("Getting the initial file position");
	strcpy(fin_stk.currit().name,name);
}

static bool pop_file() {
	if (fin && fin != stdin) {
		fflush(fin);
		fclose(fin);
	}
	fin = 0;
	fin_stk.pop();
	if (fin_stk.empty()) return fin;
	vvfilepos &pos = fin_stk.currit();
	if (!pos.name[0]) error("INTERNAL","filename is null");
	if (!strcmp(pos.name,"-")) fin = stdin;
	else {
		fin = fopen(pos.name,"rt");
		if (!fin || fsetpos(fin,&pos.pos)) error("Reopening input file",pos.name);
	}
	return fin;
}

static bool is_eof() {
	while (fin && feof(fin)) pop_file();
	return !fin;
}

const int linebuflen = 4096;
static char linebuf[linebuflen+1]; // STATIC LIMITATION

static bool vvreadline() {
	linebuf[0] = 0;
	if (!fin) return false; //error("No input file","reading line");
	while (feof(fin) || !fgets(linebuf,linebuflen,fin)) {
		if (!pop_file()) return false;
	}
	linebuf[linebuflen] = 0;
	char *p = strchr(linebuf,'\n');
	if (p) *p = 0;
	return true;
}

char getc_escape(const char *(&p)) {
	if (!*p) return 0;
	char c = *p++;
	if (c != '\\') return c;
	switch (*p++) {
	case 't': return '\t'; break;
	case 'n': return '\n'; break;
	case 'v': return '\v'; break;
	case 'r': return '\r'; break;
	case '\\': return '\\'; break;
	case 'q': return '\"'; break;
	case 'b': return ' ';
	case 0: --p; return '\\';
	default: return p[-1];
	}
}

int str_escape(char *dest, const char *src, int maxlen = 0) {
	int i = 0;
	for (; *src; ++i, ++dest) {
		if (maxlen && i >= maxlen) break;
		*dest = getc_escape(src);
	}
	*dest = 0;
	return i;
}

static int _argc = 0;
static char **_argv = 0;

//: Var object

class Var {
	virtual void clear() {};
	virtual const char *update(const char *s,bool escape = false) = 0;
public:
	Var() {}
	virtual ~Var() {clear();}
	virtual const char *get() const = 0;
	const char *set(const char *s) {return update(s);}
	const char *setesc(const char *s) {return update(s,true);}
};

class UserVar : public Var {
	const char *value;
	void clear() {
//printf("\nDeleting uservar contents: \"%s\"\n",value);fflush(stdout);
		if (value) {delete[] (const char *) value; value = 0;}
	}
	const char *update(const char *s,bool escape = false) {
//printf("\nUpdating uservar contents: from \"%s\" to \"%s\"\n",value,s);fflush(stdout);
		clear();
		char *temp = new char[strlen(s)+1];
		if (escape) str_escape(temp,s);
		else strcpy(temp,s);
//printf("\nCreating uservar contents: \"%s\"\n",temp);fflush(stdout);
		return value = temp;
	}
public:
	UserVar() : value(0) {}
	~UserVar() {clear();}
	const char *get() const {return value ? value : "";}
};

class SysStrVar : public Var {
	char *text;
	int maxlen;
	void clear() {}
	const char *update(const char *s,bool escape = false) {
		strncpy(text,s,maxlen);
		text[maxlen] = 0;
		if (escape) str_escape(text,text);
//printf("\nUpdating sysstrvar contents: \"%s\"\n",text);
		return text;
	}
public:
	SysStrVar(char *sysv,int len) : text(sysv), maxlen(len) {}
	~SysStrVar() {clear();}
	const char *get() const {return text;}
};

class SysIntVar : public Var {
	mutable char buff[16]; // STATIC LIMITATION
	bool readonly;
	int &rv;
	void clear() {}
	const char *update(const char *s,bool escape = false) {
		if (!readonly) rv = (int) strtol((char *)s,NULL,10);
		return s;
	}
public:
	SysIntVar(int &_rx,bool _readonly = false) : rv(_rx),readonly(_readonly) {}
	~SysIntVar() {clear();}
	const char *get() const {sprintf(buff,"%d",rv);return buff;}
};

class SysDoubleVar : public Var {
	mutable char buff[32]; // STATIC LIMITATION
	bool readonly;
	double &rv;
	void clear() {}
	const char *update(const char *s,bool escape = false) {
		if (!readonly) rv = strtod((char *)s,NULL);
		return s;
	}
public:
	SysDoubleVar(double &_rx,bool _readonly = false) : rv(_rx),readonly(_readonly) {}
	~SysDoubleVar() {clear();}
	const char *get() const {sprintf(buff,"%g",rv);return buff;}
};

class NamedVar : public BasicId {
	Var *pv; // *pv is always owned by NamedVar (the destructor will delete it!))
public:
	NamedVar(const char *name,bool own = false) : BasicId(name,own), pv(0) {
//printf("\nCreating NamedVar object associated to \"%s\"\n",operator const char *());
	}
	~NamedVar() {
//printf("\nDestroying NamedVar object associated to \"%s\"\n",operator const char *());
		if (pv) delete pv; pv = 0;
	}
	void fill(Var *_pv) { // Ownership of *_pv is transferred to the NamedVar object!!! 
//printf("\nTransferring Var object associated to \"%s\"\n",operator const char *());
		if (pv) error("Attempting to set twice a NamedVar object");
		pv = _pv;
	}	 
	const char *get() const {return pv->get();} // Never call it before using fill()!!!
	const char *set(const char *s) {return pv->set(s);} // Never call it before using fill()!!!
	const char *setesc(const char *s) {return pv->setesc(s);} // Never call it before using fill()!!!
	Var *getobjpointer() {return pv;}
};

//: Named var table

HashTable<NamedVar,BasicId> vartable(4096);

NamedVar *insertsysvar(const char *n,char *sysb,int maxlen) {
	NamedVar *pnv = vartable.insert(n,false); // Do not duplicate ID buffer on actual insertion (static)
	pnv->fill(new SysStrVar(sysb,maxlen));
	return pnv;
}

NamedVar *insertsysvar(const char *n,int &rx) {
	NamedVar *pnv = vartable.insert(n,false); // Do not duplicate ID buffer on actual insertion (static)
	pnv->fill(new SysIntVar(rx));
	return pnv;
}

NamedVar *insertsysvar_ro(const char *n,int &rx) {
	NamedVar *pnv = vartable.insert(n,false); // Do not duplicate ID buffer on actual insertion (static)
	pnv->fill(new SysIntVar(rx,true));
	return pnv;
}

NamedVar *insertsysvar(const char *n,double &rx) {
	NamedVar *pnv = vartable.insert(n,false); // Do not duplicate ID buffer on actual insertion (static)
	pnv->fill(new SysDoubleVar(rx));
	return pnv;
}

NamedVar *insertsysvar_ro(const char *n,double &rx) {
	NamedVar *pnv = vartable.insert(n,false); // Do not duplicate ID buffer on actual insertion (static)
	pnv->fill(new SysDoubleVar(rx,true));
	return pnv;
}

//: Some system vars

const int maxseplen = 255;
char ifsep[maxseplen+1] = ",";  // STATIC LIMITATION
char ofsep[maxseplen+1] = ",";  // STATIC LIMITATION
char orsep[maxseplen+1] = "\n";  // STATIC LIMITATION

#define	ifsep_len	(strlen(ifsep))
#define	ofsep_len	(strlen(ofsep))
#define	orsep_len	(strlen(orsep))

static double version = VER_MAJOR+0.01*VER_MINOR;

static int nr = 0;
static int rrep = 1;

static int nf;

//: Splitting input records into fields

static const char **fields; // a NULL fields must imply nf=0!!!
static int *lengths;
static bool fieldsfromregexpr;

static void vvclearfields() {
	if (fields) {
		delete[] fields;
		fields = 0;
	}
	if (lengths) {
		delete[] lengths;
		lengths = 0;
	}
	nf = 0;
	fieldsfromregexpr = false;
}

static void vvgetfields() {
	vvclearfields();
	const char *p = linebuf;
	if (*ifsep) {
		for (nf = 1;;++nf) {
			const char *p1 = strstr(p,ifsep);
			if (!p1) break;
			p = p1+ifsep_len;
		}
	} else nf = strlen(linebuf);
	if (!nf) return;
	fields = new const char *[nf];
	lengths = new int[nf];
	p = linebuf;
	if (*ifsep) {
		for (int i = 0; i < nf-1; ++i) {
			fields[i] = p;
			p = strstr(p,ifsep);
			lengths[i] = p - fields[i];
			p += ifsep_len;
		}
		fields[nf-1] = p;
		lengths[nf-1] = strlen(p);
	} else for (int i = 0; i < nf; ++i) {
		fields[i] = p++;
		lengths[i] = 1;
	}
}

//: Regular expression matching

static bool regexpr_parse_range(const char *&r,int &a,int &b) {
	bool parsinga = true;
	a = 0;
	b = 0;
	for (;*r;++r) {
		char c = *r;
		switch (c) {
		case ' ':
		case '\t':
			continue;
		case '0':
		case '1':
		case '2':
		case '3':
		case '4':
		case '5':
		case '6':
		case '7':
		case '8':
		case '9':
			if (parsinga) a = 10*a+(c-'0');
			else b = 10*b+(c-'0');
			continue;
		case ',':
			if (parsinga) parsinga = false;
			else return false;
			continue;
		case '}':
			if (parsinga) b = a;
			return true;
		}
		break;
	}
	return false;
}

const char *regexpr_predef_charclasses[] = {
	"alnum",	"[[:alpha:][:digit:]]",
	"alpha",	"[[:lower:][:upper:]]",
	"blank",	"[ \t]",
	"cntrl",	"[\x01-\x1f\x7f]",
	"digit",	"[0-9]",
	"graph",	"[[:alnum:][:punct:]]",
	"lower",	"[a-z]",
	"print",	"[[:alnum:][:punct:] ]",
	"punct",	"[]!\"#$%&'()*+,./:;<=>?@[\\^_`{|}~-]",
	"space",	"[ \t\n\r\f\v]",
	"upper",	"[A-Z]",
	"xdigit",	"[0-9a-fA-F]",
NULL};

static bool regexpr_matches_charclass(const char *&r,char c);

static bool regexpr_matches_predef_charclass(const char *&r,char c) {
	const char *p2 = strchr(r,']');
	if (!p2 || p2 == r || p2[-1] != ':') error("REGEXPR","Unmatched '[:' in predef. character class specification");
	const char *p1 = r;
	int l = p2-r-1;
	r = p2;
//fprintf(stderr,"--- Using predef. char. class \"%.*s\" ---\n",l,p1);
	for (int i = 0; regexpr_predef_charclasses[2*i]; ++i) {
		p2 = regexpr_predef_charclasses[2*i];
		if (l == strlen(p2) && !strncmp(p1,p2,l)) {
			p2 = regexpr_predef_charclasses[2*i+1]+1;
			return regexpr_matches_charclass(p2,c);
		}
	}
	error("REGEXPR","Unknown predefined character class");
	return false;
}

static bool regexpr_matches_charclass(const char *&r,char c) {
	bool inverted = false;
	bool firstchar = true;
	bool found = false;
	if (r[0] == '^') {inverted = true; ++r;}
	for (;*r;++r,firstchar=false) {
		switch (r[0]) {
		case '[':
			if (r[1] == ':') {
				if (regexpr_matches_predef_charclass(r+=2,c)) found = true; // found a match!
				continue;
			}
			break;
		case ']':
			if (!firstchar) { // c1 closes the character class specification
				if (!c) return false;
				return inverted? !found : found;
			}
			break;
		}
		if (r[1] == '-' && r[2] != ']') { // range expression
			if (c >= r[0] && c <= r[2]) found = true; // found a match!
			r+=2;
		} else { // single character
			if (c == r[0]) found = true; // found a match!
		}
	}
	error("REGEXPR","Unmatched '[' in character class specification");
	return false;
}

const int REGEXPR_MAXREFS = 32; // STATIC LIMITATION
struct regexpr_ref {
	const char *r;
	bool closed;
	int start,laststart;
	int len,lastlen;
	bool valid,lastvalid;
	regexpr_ref() : r(NULL), valid(false), lastvalid(false) {}
	void clear() {r = NULL; valid = false; lastvalid = false;}
} regexpr_refs[REGEXPR_MAXREFS];

// regexpr_refs[0] is reserved for the whole matching string

int regexpr_nrefs = 1; // index of the first free (or invalid) reference

static void regexpr_clearrefs() {
	for (int ref = 0; ref < REGEXPR_MAXREFS; ++ref) regexpr_refs[ref].clear();
	regexpr_nrefs = 1;
}

static int regexpr_getref(const char *r,int i) {
	int ref;
	for (ref = 1; ref < regexpr_nrefs; ++ref)
		if (regexpr_refs[ref].r == r) break;
	if (ref >= REGEXPR_MAXREFS) return -1;
	regexpr_ref &x = regexpr_refs[ref];
	if (ref == regexpr_nrefs) {
		++regexpr_nrefs; // It will never exceed REGEXPR_MAXREFS
		x.r = r;
	}
	x.start = i;
	x.len = -1; // the reference still was not successfully closed
	x.closed = false;
	x.valid = true;
	// now, invalidate all subsequent references (but the lastvalid value is kept)
	for (int j = ref + 1; j < regexpr_nrefs; ++j) regexpr_refs[j].valid = false;
	return ref;
}

static void regexpr_saveref(int ref,int i,bool result) {
	if (ref < 1 || ref >= REGEXPR_MAXREFS || !regexpr_refs[ref].valid) return;
	regexpr_ref &x = regexpr_refs[ref];
	x.closed = true;
	if (!result) return;
	x.len = i - x.start;
	x.lastvalid = true;
	x.laststart = x.start;
	x.lastlen = x.len;
}

static void regexpr_setbaseref(int i) {
	regexpr_ref &x = regexpr_refs[0];
	x.len = -1;
	x.closed = false;
	x.valid = true;
	x.start = i;
}

static void regexpr_savebaseref(int i,bool result) {
//fprintf(stderr,"\n--- Matching string saved!\n");
	regexpr_ref &x = regexpr_refs[0];
	x.closed = true;
	if (!result) return;
	x.len = i - x.start;
	x.lastvalid = true;
	x.laststart = x.start;
	x.lastlen = x.len;
}

static bool regexpr_matches_ref(const char *s,int ref,int &i) {
	if (ref < 1 || ref >= REGEXPR_MAXREFS) return false;
	regexpr_ref &x = regexpr_refs[ref];
//if (x.valid && x.len >= 0) fprintf(stderr,"\n --- Matching ref. %d: \"%.*s\" vs. \"%.*s\"\n",ref,x.len,s+x.start,x.len,s+i);
//else fprintf(stderr,"\n --- Trying to match ref. %d, that is invalid or unsuccessful\n",ref);
//if ((!x.valid || x.len < 0) && x.lastvalid && x.lastlen >= 0) fprintf(stderr," --- but we can still use the former value: \"%.*s\" vs. \"%.*s\"\n",x.lastlen,s+x.laststart,x.lastlen,s+i);
	if (!x.valid || !x.closed) error("REGEXPR","Invalid back reference in regular expression");
	if (x.len >= 0) {
		if (strncmp(s+x.start,s+i,x.len)) return false;
		i += x.len;
		return true;
	} else {
		if (!x.lastvalid || x.lastlen < 0 || strncmp(s+x.laststart,s+i,x.lastlen)) return false;
		i += x.lastlen;
		return true;
	}
}

static void regexpr_printrefs(const char *s) {
	fprintf(stderr,"\nREGEXPR REFERENCES:\n");
	fprintf(stderr,"\n  The string is \"%s\"\n",s+regexpr_refs[0].start);
	for (int ref = 0; ref < REGEXPR_MAXREFS; ++ref) {
		regexpr_ref &x = regexpr_refs[ref];
		fprintf(stderr,"  %2s [%d]: ",ref == regexpr_nrefs ? "**" : "",ref);
		if (!x.valid) fprintf(stderr,"(not valid)");
		else if (!x.closed) fprintf(stderr,"(not closed)");
		else if (x.len < 0) fprintf(stderr,"(not matched)");
		else fprintf(stderr,"\"%.*s\"",x.len,s+x.start);
		if ((!x.valid || !x.closed || x.len < 0) && x.lastvalid && x.lastlen >= 0)
			fprintf(stderr," but formerly it was \"%.*s\"",x.lastlen,s+x.laststart);
		fprintf(stderr,"\n");
 
	}
	fprintf(stderr,"\n");
}

// store duplicates of the template and the string.
// otherwise, vvprint could overwrite them!!!
char *regexpr_string = NULL;
char *regexpr_templ = NULL;

static void regexpr_storeresult(const char *t,const char *s) {
	if (regexpr_string) {
		delete[] regexpr_string;
		regexpr_string = NULL;
	}
	if (regexpr_templ) {
		delete[] regexpr_templ;
		regexpr_templ = NULL;
	}
	if (!t || !s) error("REGEXPR","Internal: null pointer(s) in call to storeresult()"); 
	regexpr_string = new char[strlen(s)+1];
	if (!regexpr_string) error("REGEXPR","Unable to create a copy of the string");
	strcpy(regexpr_string,s);
	regexpr_templ = new char[strlen(t)+1];
	if (!regexpr_templ) error("REGEXPR","Unable to create a copy of the string");
	strcpy(regexpr_templ,t);
}

// The backtrack point structure stores the internal state at the backtrack point, and it also implements the execution stack
// via a linked list: the execution stack does not delete or overwrite any previous history, but every entry keeps a pointer to
// the previous execution state (from the point of view of recursion)
struct regexpr_backtrackpoint {
	int str_pos;
	int str_pos_group;
	const char *regexpr_pos;
	char type;
	int matchcount;
	bool skipping;
	bool failed;
	int ref;
	int parent;
	regexpr_backtrackpoint() : str_pos(0), str_pos_group(0), regexpr_pos(0), type(0), matchcount(0), skipping(false), failed(false), ref(-1), parent(-1) {}
	regexpr_backtrackpoint(int i, int i0, const char *r, char t, int c, bool sk, bool f, int rf, int p) :
		str_pos(i), str_pos_group(i0), regexpr_pos(r), type(t), matchcount(c), skipping(sk), failed(f), ref(rf), parent(p) {}
};

DynStack<regexpr_backtrackpoint> regexpr_stack("REGEXPR Backtrack Points Stack",10,10);

static void regexpr_push(int i, int i0, const char *r, char type, int count = 0, bool skipping = false, bool failed = false, int ref = -1, int parent = -1) {
	regexpr_stack.push(regexpr_backtrackpoint(i,i0,r,type,count,skipping,failed,ref,parent));
//fprintf(stderr,"--- PUSHED #%d '%c' (pos=%d,inipos=%d,\"%s\",%d match(es),%sskipping,%sfailed) ref=%d parent=%d\n",regexpr_stack.curridx(),type,i,i0,r,count,skipping?"":"not ",failed?"":"not ",ref,parent);
}

// Regexpr matching test: *pr is the regular expression, and s the string to be tested, *pi is the initial position in s.
// In case of matching, *pi is updated to the index of the first unmatched character.
// *pr is always updated to the end of the matched pattern portion, limited either by ')', '/' or by a NULL character.
// if skipping then the pattern terms are parsed but not matched
// The OR and repetition commands are implemented via a conditional failure flag (fail) and a counter (matchcount).
// The conditional failure is consolidated after processing the optional repetition commands
//   failed ==> skipping
//   (failed == true) means that the template or the current choice in an OR template did not match
//   (!failed && skipping) means that a previous choice in an OR template already matched
//   (!skipping) is the normal state, when we still have chances for the template, or the current choice in an OR template, matching
// The repetition commands cannot be nested in a meaningful way:
//   a+?, a*?, a**, a+*, a*+, a?+, a?* are the same as a*, a?? is the same as a?, a++ is the same as a+.
//   a{2}{2} is the same as a{2}, and not a{4} (you would write (a{2}){2} instead).
//   a{2}{1} and a{2}{3} are also the same as a{2}.
//   a{,n}{,m} is the same as a{,n}, whereas a{n,}{m,} is the same as a{l,} for l = min(n,m). 
static bool regexpr_matches_at_ofs(const char *s,const char **pr,int *pi,bool skipping) {
//fprintf(stderr,"MATCHING%s \"%s\" FOR PATTERN \"%s\" ... ",skipping?" (SKIPPING)":"",s+*pi,*pr);
	int parent = -1; // implements the execution stack via a linked list of states
	bool fail = false; // the current pattern term matching operation failed, but still it can be ignored by '*', '+' or '?'
	bool failed = false; // some previous pattern terms matching failed, and this falsifies the whole pattern block
	// failed must imply skipping!!!
	bool result; // some temporary variable
	int ref; // the current back reference identifier
	int matchcount = 0; // counter of the number of repeated matches of the current term (implements '+')
	int i = *pi; // current position in the string s
	int lasti = i; // last tested string position
	int i0 = i; // starting string position for the current group (parenthesized subexpression)
	const char *r = *pr; // current pattern term
	const char *lastr = r; // last tested pattern term (implements one-level backtracking for '*', '+' and '?')
	for (;; ++r) {
//fprintf(stderr,"Matching \"%s\" with \"%s\"%s\n",s+i,r,fail||skipping?" (skipping)":"");
		if (!*r || *r == '/') { // '/' is reserved for appending a template to the regexpr
			if (fail) failed = skipping = true; // Consolidate the failure of the last pattern term matching
			fail = false;
			if (parent >= 0) error("REGEXPR","Unmatched '(' in regular expression");
			if (!failed) {
				*pi = i;
				*pr = r;
//fprintf(stderr,"OK\n");
//regexpr_printrefs(s);
				return true;
			}
			while (true) {
				if (regexpr_stack.empty()) { // No backtrack is possible
					*pr = r;
//fprintf(stderr,"FAIL\n");
//regexpr_printrefs(s);
					return false;
				}
				int idx = regexpr_stack.curridx();
				regexpr_backtrackpoint &x = regexpr_stack.pop();
				switch (x.type) {
				case '(':
					parent = x.parent;
					ref = x.ref;
					matchcount = x.matchcount;
					skipping = x.skipping;
					failed = x.failed;
//fprintf(stderr,"--- BACKTRACKING #%d '%c' (pos=%d,inipos=%d,\"%s\") parent=%d\n",idx,x.type,x.str_pos,x.str_pos_group,x.regexpr_pos,x.parent);
					continue;
				case '?':
					i = lasti = x.str_pos;
					i0 = x.str_pos_group;
					r = lastr = x.regexpr_pos;
					parent = x.parent;
					ref = x.ref;
					matchcount = x.matchcount;
					fail = true; // it was false at pushing time.
					skipping = failed = false;
//fprintf(stderr,"--- BACKTRACKING #%d '%c' (pos=%d,inipos=%d,\"%s\") parent=%d\n",idx,x.type,i,i0,r,parent);
					--r; // Compensates the ++r in the for loop
					break;
				case '|':
					i = lasti = i0 = x.str_pos_group;
					r = lastr = x.regexpr_pos;
					parent = x.parent;
					ref = x.ref;
					matchcount = x.matchcount;
					fail = false;
					skipping = failed = false;
//fprintf(stderr,"--- BACKTRACKING #%d '%c' (pos=%d,inipos=%d,\"%s\") parent=%d\n",idx,x.type,i,i0,r,parent);
					// Here, we do not need the --r, since we do not want the '|' to be rescanned.
					break;
				default: error("REGEXPR","Internal: Attempting to use an uninitialized backtrack point from the stack");
				}
				break;
			}
			continue;
		}
		// firstly, process unary postfix operators ('*', '+',  '?' and "{...}") because they have effects on the last pattern term
		// arbitrary non-parenthesized concatenation of repetition operations can cause undocumented behaviour (e.g. "++" or "*+?")
		switch (*r) {
		case '?':
			// if the last operation was the first one to fail, just ignore it and restore the current position at the string
			if (!skipping && !fail) regexpr_push(lasti,i0,r,'?',matchcount,skipping,failed,ref,parent);
			if (!skipping && fail) {fail = false; i = lasti;}
			continue;
		case '*':
			// if the last operation was the first one to fail, just ignore it and restore the current position at the string
			// but if nothing failed, retry the last term matching (unless it is an empty-string term like '^' or '$', to avoid endless loops)
			if (!skipping && !fail) regexpr_push(lasti,i0,r,'?',matchcount,skipping,failed,ref,parent);
			if (!skipping) {
				if (fail) {fail = false; i = lasti;}
				else if (i > lasti) {r = lastr; lasti = i; ++matchcount; break;}
			}
			continue;
		case '+':
			// if the last operation was the first one to fail and it is not the first attempt, just ignore it and restore the current position at the string
			// if nothing failed, retry the last term matching (unless it is an empty-string term like '^' or '$', to avoid endless loops)
			if (!skipping && !fail && matchcount > 0) regexpr_push(lasti,i0,r,'?',matchcount,skipping,failed,ref,parent);
			if (!skipping) {
				if (fail && matchcount > 0) {fail = false; i = lasti;} // the same thing as in '?', but not if the first matching attempt fails
				if (!fail && i > lasti) {r = lastr; lasti = i; ++matchcount; break;} // try again (but avoid endless loops caused by '$' and '^')
			}
			continue;
		case '{':
			{
				int a,b;
				if (!regexpr_parse_range(++r,a,b)) error("REGEXPR","Wrong range specification '{m,n}' in regular expression");
				// b == 0 is interpreted as 'no b is specified', and then there is no upper limit in the repeated matchings
				if (a > b && b > 0) error("REGEXPR","In a range specification '{m,n}' m cannot be greater than n");
				// if the last operation was the first one to fail and there were enough successful previous matching attempts, just ignore it and 
				// restore the current position at the string
				// if nothing failed and there are not too many successful matching attempts, then retry the last term matching (unless it is 
				// an empty-string term like '^' or '$', to avoid endless loops)
				if (!skipping && !fail && matchcount >= a) regexpr_push(lasti,i0,r,'?',matchcount,skipping,failed,ref,parent);
				if (!skipping) {
					if (fail && matchcount >= a) {fail = false; i = lasti;} // the same thing as in '?', if there were enough successful matchings
					if (!fail && i > lasti && (++matchcount < b || b == 0)) {r = lastr; lasti = i; break;} // try again (but avoid endless loops)
				}
			}
			continue;
//		case '}': error("REGEXPR","Unmatched '}' in regular expression");
		default:
			// store current string position and current pattern term
			lastr = r; lasti = i; matchcount = 0;
		}
		if (fail) failed = skipping = true; // Consolidate the failure and start skipping the remaining of the template
		fail = false;
		char c = *r;
		switch (c) {
		case '^': if (!skipping && i != 0) fail = true; continue; 
		case '$': if (!skipping && s[i]) fail = true; continue;
		case '(':
			i0 = i;
			ref = regexpr_getref(lastr,i);
			regexpr_push(i,i0,r,'(',matchcount,skipping,failed,ref,parent);
			parent = regexpr_stack.curridx();
//if (*r != ')') error("REGEXPR","Unmatched '(' in regular expression");
			continue;
		case ')':
			{
				result = !failed;
if (parent < 0) error("REGEXPR","Internal: Attempting to access a backtrack point with index < 0");
				regexpr_backtrackpoint x = regexpr_stack[parent];
				ref = x.ref;
				lasti = x.str_pos;
				i0 = x.str_pos_group;
				lastr = x.regexpr_pos;
				matchcount = x.matchcount;
				skipping = x.skipping;
				failed = x.failed;
//fprintf(stderr,"--- RETURNING #%d (pos=%d,inipos=%d,\"%s\") parent=%d\n",parent,i,i0,r,x.parent);
				parent = x.parent;
				if (!skipping && !result) fail = true;
				regexpr_saveref(ref,i,!skipping && result);
//fprintf(stderr,"%s\n",failed?"FAIL":"OK");
				continue;
			}
		case '|':
			if (!skipping) regexpr_push(i0,i0,lastr,'|',matchcount,skipping,failed,ref,parent);
			if (skipping) { // either a previous choice matched or the current choice failed to match
			        if (failed) { // current choice did not match: try next one
			        	failed = skipping = false;
					i = lasti = i0; // restore the initial string position
				}
			} else skipping = true; // the current choice is the first matching one: skip the remaining choices
			continue;
		case '[':
			result = regexpr_matches_charclass(++r,s[i]);
//			if (*r != ']') error("REGEXPR","INTERNAL: Unmatched '[' in char class specification"); // already tested inside the function
			if (!skipping && !result) fail = true;
			if (!skipping && result && s[i]) ++i;
			continue;
//		case ']': error("REGEXPR","Unmatched ']' in regular expression");
		case '.':
			if (!skipping) {
				if (!s[i]) fail = true;
				else ++i;
			}
			continue;
		case '\\':
			c = *++r;
			if (c >= '1' && c <= '9') {
				if (!skipping && !regexpr_matches_ref(s,c-'0',i)) fail = true;
				continue;
			}
			switch (c) {
			case 'w':
				{
					const char *p = "[_[:alnum:]]";
					result = regexpr_matches_charclass(++p,s[i]);
					if (!skipping && !result) fail = true;
					if (!skipping && result && s[i]) ++i;
					continue;
				}
			case 'W':
				{
					const char *p = "[^_[:alnum:]]";
					result = regexpr_matches_charclass(++p,s[i]);
					if (!skipping && !result) fail = true;
					if (!skipping && result && s[i]) ++i;
					continue;
				}
			case 'n': c = '\n'; break;
			case 't': c = '\t'; break;
			case 'b': c = ' '; break;
			case '\\':
			case '^':
			case '$':
			case '|':
			case '*':
			case '?':
			case '+':
			case '.':
			case '(':
			case ')':
			case '[':
			case ']':
			case '{':
			case '}':
			case '/':
				break;
			case 0: error("REGEXPR","Unfinished escape sequence in regular expression");
			default: error("REGEXPR","Bad escape sequence in regular expression");
			}
		}
		if (skipping) continue;
		if (s[i] != c) fail = true;
		else if (s[i]) ++i;
	}
	return false;
}

static bool regexpr_matches(const char *s,const char *r) {
	if (fieldsfromregexpr) {
		vvclearfields(); // Remove the association of previous back references and input fields
		fieldsfromregexpr = false;
	}
	bool result = false;
	const char *r0 = r;
	int i = 0;
	while (true) {
		regexpr_stack.flush();
		regexpr_clearrefs();
		r = r0;
		regexpr_setbaseref(i);
		result = regexpr_matches_at_ofs(s,&r,&i,false);
		if (*r && *r != '/') error("REGEXPR","Unmatched ')' in regular expression");
		regexpr_savebaseref(i,result);
		if (result || !s[i++]) break;
	}
	regexpr_storeresult(*r? r+1 : "\\0",s);
	return result;
}

// Deal back references as input fields
static void dump_backrefs_as_fields() {
	vvclearfields();
	fieldsfromregexpr = true;
	for (nf = 0;nf < REGEXPR_MAXREFS;++nf) {
		regexpr_ref &x = regexpr_refs[nf];
		if (!x.valid || !x.closed) break;
	}
	fields = new const char *[nf];
	lengths = new int[nf];
	for (int i = 0; i < nf; ++i) {
		regexpr_ref &x = regexpr_refs[i];
		if (x.len >= 0) {
			fields[i] = regexpr_string + x.start;
			lengths[i] = x.len;
		} else if (x.lastvalid && x.lastlen >= 0) {
			fields[i] = regexpr_string + x.laststart;
			lengths[i] = x.lastlen;
		} else { // manage unmatched back refs as zero length strings
			fields[i] = regexpr_string; // dummy yet valid value
			lengths[i] = 0;
		}
	}	
}

// Split input record according to a regexpr as field separator.
// Store also the matching individual separators as fields.
// Zero-length field + separator would cause an endless loop.
// In this case, the field is assumed to be a character.
static void vvgetfields_regexpr(const char *r) {
	regexpr_ref &x = regexpr_refs[0];
	vvclearfields();
	const char *p = linebuf;
	for (nf = 1;*p;++nf) {
		if (!regexpr_matches(p,r)) break;
		p += x.start + x.len;
		if (x.start + x.len <= 0) ++p; // avoid endless loops!
	}
	nf = 2*nf - 1;
	fields = new const char *[nf];
	lengths = new int[nf];
	p = linebuf;
	for (int i = 0; i < nf - 1; i += 2) {
		fields[i] = p;
		if (!regexpr_matches(p,r)) error("GETFIELDS","Internal: regexpr should match here");
		if (x.start + x.len > 0) { // avoid endless loops!
			lengths[i] = x.start;
			fields[i+1] = p + x.start;
			lengths[i+1] = x.len;
			p += x.start + x.len;
		} else {
			lengths[i] = 1;
			fields[i+1] = p + 1;
			lengths[i+1] = 0;
			++p;
		}
	}
	fields[nf-1] = p;
	lengths[nf-1] = strlen(p);
}

//: Template expansion

const int expbuflen = 65535;
static char expbuf[expbuflen+1]; // STATIC LIMITATION
static int exppos = 0;
static int skipping = -1;
static int enable = 1; // implements breaking the expansion of the outer template
static int loop = 0; // implements looping in outer template expansion
enum { // 1...9 are reserved for fields #1,...,#9. Commands correspond to negative values.
	COMM_ALLFIELDS=-1, COMM_NR=-2, COMM_FIELDRANGE=-3, COMM_ARGVRANGE=-4,
	COMM_NEXTREC=-8, COMM_SAMEREC=-9, COMM_SAMEREC_RE=-10, COMM_SETREC=-11, COMM_PUSHFILE=-12, COMM_POPFILE=-13, COMM_CHECKFILE=-14,
	COMM_SETINCHANNEL0=-15, COMM_SETINCHANNEL1=-16, COMM_SETINCHANNEL2=-17, COMM_SETINCHANNEL3=-18,
	COMM_SETINCHANNEL4=-19, COMM_SETINCHANNEL5=-20, COMM_SETINCHANNEL6=-21, COMM_SETINCHANNEL7=-22,
	COMM_SETINCHANNEL8=-23, COMM_SETINCHANNEL9=-24,
	COMM_IFMATH=-33, COMM_TOGGLESKIP=-34, COMM_ENDSKIP=-35, COMM_PUSHMARK=-36, COMM_BREAK=-37, COMM_LOOP=-38,
	COMM_FLUSH=-41,
	COMM_SAVEVAR=-49, COMM_LOADVAR=-50, COMM_EXECVAR=-51, COMM_CALCVAR=-52, COMM_EXEC=-53,
	COMM_REGEXPR=-56,
	COMM_OP=-64, // numbers below COMM_OP are reserved for comparison operators.
	COMM_NOP=-127 // numbers below COMM_NOP do nothing.
};
enum {OP_EQUAL=1,OP_SUBSTR=2,OP_SUPERSTR=3,OP_PREFIX=4,OP_POSTFIX=5,OP_MATCHES=6}; // Actual command numbers are COMM_OP-OP_...

DynStack<int> mark_stack("Template Expansion Mark Stack",64,64);
DynStack<int> skip_stack("Template Expansion Skip Stack",64,64);

void vvflush() {
	if (exppos) printf("%.*s",exppos,expbuf);
	exppos = 0;
}

void vvprint(const char *s, int l = -1) { // printing into expbuf does not append the NULL terminator character!
	if (!s) return;
	if (skipping >= 0) return;
	if (l < 0) l = strlen(s);
	if (exppos+l > expbuflen) error("expansion buffer overflow");
	strncpy(expbuf+exppos,s,l);
	exppos += l;
}

static bool checkformat(const char *fmt, const char *types) {
	if (!fmt || *fmt != '%') return false;
	const char *p = fmt+1;
	p += strspn(p,"#+- 0");
	if (*p=='*') return false;
	p += strspn(p,"0123456789");
	if (*p=='.') {
		if (*++p=='*') return false;
		p += strspn(p,"0123456789");
	}
//	if (strchr("lL",*p)) p++;
	if (!strchr(types,*p)) return false;
	return !p[1];
}

void vvprint(int i, const char *fmt = 0) {
	static char buf[16]; // STATIC LIMITATION
	sprintf(buf,checkformat(fmt,"iduoxX") ? fmt : "%d",i);
	vvprint(buf);
}

void vvprint(double v, const char *fmt = 0) {
	static char buf[32]; // STATIC LIMITATION
	if (checkformat(fmt,"iduoxX")) sprintf(buf,fmt,(int) v);
	else sprintf(buf,checkformat(fmt,"eEfgG") ? fmt : "%g",v);
	vvprint(buf);
}

static void vvprintfsep() {vvprint(ofsep,ofsep_len);}

static void vvprintrsep() {vvprint(orsep,orsep_len);}

static void vvprintfield(int i) {
	if (i <= 0 || i > nf) return; // undefined fields are ignored
	if (i == nf) vvprint(fields[nf-1],lengths[nf-1]);
	else vvprint(fields[i-1],lengths[i-1]);
}

static void vvprintfields() {
	if (!nf) return;
	for (int i = 0; i < nf - 1; ++i) {
		vvprint(fields[i],lengths[i]);
		vvprintfsep();
	}
	vvprint(fields[nf-1],lengths[nf-1]);
}

// return -1 if conditional operator evaluates to true, 1 to false, and 0 if it is not evaluated.
int comm_op(int i) {
	expbuf[exppos] = 0;
	const char *p2 = expbuf+mark_stack.pop()+1;
	exppos = mark_stack.currit();
	const char *p1 = expbuf+mark_stack.pop()+1;
	if (skipping >= 0) return 0;
	switch (i) {
		case OP_EQUAL: return !strcmp(p1,p2) ? -1 : 1;
		case OP_SUBSTR: return strstr(p2,p1) ? -1 : 1;
		case OP_SUPERSTR: return strstr(p1,p2) ? -1 : 1;
		case OP_PREFIX: return !strncmp(p1,p2,strlen(p1)) ? -1 : 1;
		case OP_POSTFIX: return (strlen(p1)<=strlen(p2)) && (!strcmp(p1,p2+strlen(p2)-strlen(p1))) ? -1 : 1;
		case OP_MATCHES: return regexpr_matches(p1,p2) ? -1 : 1;
		default: error("INTERNAL","Unknown conditional operator found while expanding template");
	}	
}

void comm_flush() {
	expbuf[exppos] = 0;
	exppos = mark_stack.currit();
	const char *p = expbuf+mark_stack.pop()+1;
	if (skipping < 0) printf("%s",p);
}

void comm_getvar() {
	expbuf[exppos] = 0;
	exppos = mark_stack.currit();
	const char *p = expbuf+mark_stack.pop()+1;
	if (skipping < 0) {
		NamedVar *pvar = vartable.find(p);
		if (pvar) vvprint(pvar->get());
	}
}

void comm_setvar() {
	expbuf[exppos] = 0;
	const char *p2 = expbuf+mark_stack.pop()+1;
	exppos = mark_stack.currit();
	const char *p1 = expbuf+mark_stack.pop()+1;
	if (skipping < 0) {
		NamedVar *pvar = vartable.insert(p1,true); // Duplicate ID buffer on actual insertion
		if (!pvar->getobjpointer()) {
			pvar->fill(new UserVar);
		}
		pvar->set(p2);
	}
}

static void expanddirecttemplate(const char *p);

void comm_execvar() {
	expbuf[exppos] = 0;
	exppos = mark_stack.currit();
	const char *p = expbuf+mark_stack.pop()+1;
	if (skipping < 0) {
		NamedVar *pvar = vartable.find(p);
		if (pvar) expanddirecttemplate(pvar->get());
	}
}

void comm_exec() {
	expbuf[exppos] = 0;
	exppos = mark_stack.currit();
	const char *p = expbuf+mark_stack.pop()+1;
	if (skipping < 0) {
		expanddirecttemplate(p);
	}
}

// expands to template (if result is true) and detects the option '|' that dumps back references as input fields
static void regexpr_expand(bool result) {
//fprintf(stderr,"\n--- expanding regexpr template \"%s\" for string \"%s\" and result = %s ---\n",regexpr_templ,regexpr_string,result ? "true" : "false");
	if (!regexpr_string || !regexpr_templ) error("REGEXPR","Unable to expand template: no string or template specified");
	const char *t = regexpr_templ;
	while (*t) {
		char c = *t++;
		if (c == '/') break;
		if (c == '\\') {
			c = *t++;
			if (c >= '0' && c <= '9') {
				if (result) {
//fprintf(stderr,"\n    printing back ref %d\n",c-'0');
					regexpr_ref &x = regexpr_refs[c-'0'];
					if (x.valid && x.closed && x.len >= 0)
						vvprint(regexpr_string+x.start,x.len);
					else if (x.lastvalid && x.lastlen >= 0)
						vvprint(regexpr_string+x.laststart,x.lastlen);
// Do no detect errors and just ignore unmatched back references!
// This is better, because some strings matching a complex regular expressions can cause some valid back references to be undefined.
//					else
//						error("REGEXPR","Invalid or unmatched back reference in template");
				}
				continue;
			}
			switch (c) {
			case '\\': break;
			case '/': break;
			case 'n': c = '\n'; break;
			case 't': c = '\t'; break;
			default: error("REGEXPR","Illegal escaped char in template");
			}
		}
		if (result) vvprint(&c,1);
	}
	if (*t == '|') dump_backrefs_as_fields();
}

void comm_regexpr() {
	expbuf[exppos] = 0;
	const char *p2 = expbuf+mark_stack.pop()+1;
	exppos = mark_stack.currit();
	const char *p1 = expbuf+mark_stack.pop()+1;
	if (skipping < 0) 
		regexpr_expand(regexpr_matches(p1,p2));
}

// This command sets the contents of the input record from the expansion buffer and rescans the fields
void comm_setrecord() {
	expbuf[exppos] = 0;
	exppos = mark_stack.currit();
	const char *p = expbuf+mark_stack.pop()+1;
	if (skipping < 0) {
		strncpy(linebuf,p,linebuflen);
		linebuf[linebuflen]=0;
		vvgetfields();
	}
}

void comm_samerecord_re() {
	expbuf[exppos] = 0;
	exppos = mark_stack.currit();
	const char *p = expbuf+mark_stack.pop()+1;
	if (skipping < 0) vvgetfields_regexpr(p);
}

void comm_pushfile() {
	expbuf[exppos] = 0;
	exppos = mark_stack.currit();
	const char *p = expbuf+mark_stack.pop()+1;
	if (skipping < 0) push_file(p);
}

void comm_calcvar() {
	expbuf[exppos] = 0;
	exppos = mark_stack.currit();
	int exppos0 = exppos;
	const char *p = expbuf+mark_stack.pop()+1;
	if (skipping >= 0) return;
	bool mute = false;
	if (*p=='@') {
		mute = true;
		p++;
	}
	switch (*p) {
	case '&':
		{
			NamedVar *pvar = vartable.find(++p);
			if (!pvar) error("Unknown varname in #$[&varname]");
			vvprint(pvar->get());
			expbuf[exppos] = 0;
			p = expbuf+exppos0;
			break;
		}
	case '`':
		{
			NamedVar *pvar = vartable.find(++p);
			if (!pvar) error("Unknown varname in #$[`varname]");
			expanddirecttemplate(pvar->get());
			expbuf[exppos] = 0;
			p = expbuf+exppos0;
			break;
		}
	}
	const char *fmt = strchr(p,';');
	if (fmt) expbuf[(fmt++)-expbuf] = 0; // replace ';' by a null character
	double v = vexpr(p);
	exppos = exppos0;
	expbuf[exppos] = 0;
	if (!mute) vvprint(v,fmt);
}

// return -1 if conditional operator evaluates to true, 1 to false, and 0 if it is not evaluated.
int comm_ifmath() {
	expbuf[exppos] = 0;
	exppos = mark_stack.currit();
	int exppos0 = exppos;
	const char *p = expbuf+mark_stack.pop()+1;
	if (skipping >= 0) return 0;
	bool mute = false;
	if (*p=='@') {
		mute = true;
		p++;
	}
	switch (*p) {
	case '&':
		{
			NamedVar *pvar = vartable.find(++p);
			if (!pvar) error("Unknown varname in #$[&varname]");
			vvprint(pvar->get());
			expbuf[exppos] = 0;
			p = expbuf+exppos0;
			break;
		}
	case '`':
		{
			NamedVar *pvar = vartable.find(++p);
			if (!pvar) error("Unknown varname in #$[`varname]");
			expanddirecttemplate(pvar->get());
			expbuf[exppos] = 0;
			p = expbuf+exppos0;
			break;
		}
	}
	const char *fmt = strchr(p,';');
	if (fmt) expbuf[(fmt++)-expbuf] = 0; // replace ';' by a null character
	double v = vexpr(p);
	exppos = exppos0;
	expbuf[exppos] = 0;
	return v==0?1:-1;
}

static void comm_fieldrange() {
	expbuf[exppos] = 0;
	exppos = mark_stack.currit();
	const char *p0 = expbuf+mark_stack.pop()+1;
	if (skipping >= 0) return;
	char *buf = new char[strlen(p0)+1]; // We need a copy here because otherwise expanded text would overwrite the pattern!
	strcpy(buf,p0);
	try {
		int i,mini,maxi,incr;
		char *endp;
		int j = 0;
		const char *p = buf;
		bool firstone = true;
		int n;
		for (;;) {
			i = strtol(p,&endp,10);
			if (p==endp && *endp) error("bad field range specification: missing number",p);
			p = endp;
			switch (j) {
			case 0: mini = i; incr = 1; maxi = mini; break;
			case 1: maxi = i; break;
			default:
				incr = maxi; maxi = i;
				if (!incr) error("bad field range specification: the increment y in x:y:z must be nonzero");
			}
			j++;
			switch (*p) {
			case ':':
				if (j>2) error("bad field range specification: `:' is a binary or ternary operator",p);
				p++;
				continue;
			case ',':
			case 0:
				if (mini <= 0) mini += nf;
				if (maxi <= 0) maxi += nf;
				n = (maxi-mini)*incr >= 0 ? (maxi-mini)/incr+1 : 0; // the number of elements in the range
				if (firstone && n) {
					vvprintfield(mini);
					firstone = false;
					mini += incr;
					n--;
				}
				for (i=0;i<n;i++) {
					vvprintfsep();
					vvprintfield(mini+i*incr);
				}
				if (!*p) break;
				p++;
				j=0;
				continue;
			default:
				error("bad field range specification: unexpected character",*p);
			}
			break;
		}
	} catch (...) {
		if (buf) delete[] buf;
		throw;
	}
	if (buf) delete[] buf;
}

static void comm_argvrange() {
	expbuf[exppos] = 0;
	exppos = mark_stack.currit();
	const char *p0 = expbuf+mark_stack.pop()+1;
	if (skipping >= 0) return;
	char *buf = new char[strlen(p0)+1]; // We need a copy here because otherwise expanded text would overwrite the pattern!
	strcpy(buf,p0);
	try {
		int i,mini,maxi,incr;
		char *endp;
		int j = 0;
		const char *p = buf;
		bool firstone = true;
		int n;
		for (;;) {
			i = strtol(p,&endp,10);
			if (p==endp && *endp) error("bad argv range specification: missing number",p);
			p = endp;
			switch (j) {
			case 0: mini = i; incr = 1; maxi = mini; break;
			case 1: maxi = i; break;
			default:
				incr = maxi; maxi = i;
				if (!incr) error("bad argv range specification: the increment y in x:y:z must be nonzero");
			}
			j++;
			switch (*p) {
			case ':':
				if (j>2) error("bad argv range specification: `:' is a binary or ternary operator",p);
				p++;
				continue;
			case ',':
			case 0:
				if (mini < 0) mini += _argc;
				if (maxi < 0) maxi += _argc;
				n = (maxi-mini)*incr >= 0 ? (maxi-mini)/incr+1 : 0; // the number of elements in the range
				if (firstone && n) {
					if (mini >= 0 && mini < _argc) vvprint(_argv[mini]);
					firstone = false;
					mini += incr;
					n--;
				}
				for (i=0;i<n;i++) {
					vvprintfsep();
					if (mini+i*incr >= 0 && mini+i*incr < _argc) vvprint(_argv[mini+i*incr]);
				}
				if (!*p) break;
				p++;
				j=0;
				continue;
			default:
				error("bad argv range specification: unexpected character",*p);
			}
			break;
		}
	} catch (...) {
		if (buf) delete[] buf;
		throw;
	}
	if (buf) delete[] buf;
}

static void vvprintcomm(int i) {
	if (i <= COMM_NOP) return;
	if (i <= COMM_OP) {
		skip_stack.push(skipping);
		skipping = comm_op(COMM_OP-i)*enable; // enable=0 implies skipping the expansion of any element
		return;
	}
	if (i <= 0) {
		switch (i) {
		case COMM_ALLFIELDS: vvprintfields(); break;
		case COMM_NR: vvprint(nr); break;
		case COMM_IFMATH: 
			skip_stack.push(skipping);
			skipping = comm_ifmath()*enable; // enable=0 implies skipping the expansion of any element
			break;
		case COMM_TOGGLESKIP: skipping = -skipping; break;
		case COMM_ENDSKIP: skipping = skip_stack.pop()*enable; break; // enable=0 implies skipping the expansion of any element
		case COMM_PUSHMARK:
			mark_stack.push(exppos);
			if (exppos+1 > expbuflen) error("expansion buffer overflow");
			expbuf[exppos++] = 0;
			break;
		case COMM_NEXTREC: // This command scans the next record (even if it is special!) and continues the expansion procedure
			if (skipping < 0 && vvreadline()) vvgetfields();
			break;
		case COMM_SAMEREC: // This command rescans the fields of the current line (perhaps with a different separator)
			if (skipping < 0) vvgetfields();
			break;
		case COMM_SETREC: comm_setrecord(); break;
		case COMM_SAMEREC_RE: comm_samerecord_re(); break;
		case COMM_PUSHFILE: comm_pushfile(); break;
		case COMM_POPFILE: if (skipping < 0) pop_file(); break;
		case COMM_CHECKFILE: if (skipping < 0) vvprint(is_eof()?"0":"1"); break;
		case COMM_SETINCHANNEL0: if (skipping < 0) in_channel=0; break;
		case COMM_SETINCHANNEL1: if (skipping < 0) in_channel=1; break;
		case COMM_SETINCHANNEL2: if (skipping < 0) in_channel=2; break;
		case COMM_SETINCHANNEL3: if (skipping < 0) in_channel=3; break;
		case COMM_SETINCHANNEL4: if (skipping < 0) in_channel=4; break;
		case COMM_SETINCHANNEL5: if (skipping < 0) in_channel=5; break;
		case COMM_SETINCHANNEL6: if (skipping < 0) in_channel=6; break;
		case COMM_SETINCHANNEL7: if (skipping < 0) in_channel=7; break;
		case COMM_SETINCHANNEL8: if (skipping < 0) in_channel=8; break;
		case COMM_SETINCHANNEL9: if (skipping < 0) in_channel=9; break;
		case COMM_FLUSH: comm_flush(); break;
		case COMM_LOADVAR: comm_getvar(); break;
		case COMM_SAVEVAR: comm_setvar(); break;
		case COMM_EXECVAR: comm_execvar(); break;
		case COMM_CALCVAR: comm_calcvar(); break;
		case COMM_EXEC: comm_exec(); break;
		case COMM_REGEXPR: comm_regexpr(); break;
		case COMM_FIELDRANGE: comm_fieldrange(); break;
		case COMM_ARGVRANGE: comm_argvrange(); break;
		case COMM_BREAK: // breaks expansion (by skipping everything)
			if (skipping < 0) skipping = enable = 0;
			break;
		case COMM_LOOP: // breaks expansion and repeats it (if in the body of the outer template)
			if (skipping < 0) {
				skipping = enable = 0;
				loop = 1;
			}
			break;
		default: error("bad field index value");
		}
		return;
	}
	vvprintfield(i);
}

//: Reading templates

static FILE *ftempl = 0;

const int templbuflen = 65535;
static char templbuf[templbuflen+1];

int headid = -1;
int bodyid = 0;
int tailid = -1;
const int maxntf = 256;
static int ntf = 0;
static const char *tfields[maxntf+1];
static int tcomm[maxntf];

DynStack<int> stat("Template Parser Status Stack",64,64);

enum {STAT_INIT=0,STAT_IFCOND,STAT_IFMATH,STAT_IFTRUE,STAT_IFFALSE,STAT_FLUSH,STAT_GETVAR,STAT_SETVAR,STAT_SETVARVAL,STAT_EXECVAR,STAT_EXEC,STAT_REGEXPRSTR,STAT_REGEXPREXP,STAT_MATH,STAT_FIELDRANGE,STAT_ARGVRANGE,STAT_SETRECORD,STAT_SAMEREC_RE,STAT_PUSHFILE};

// Finite state machine for [...] blocks
//
//	STATE		PARSED	PRODUCES	NEXT STATE
//
//	Init    	#?[	COMM_PUSHMARK	STAT_IFCOND
//	STAT_IFCOND	]<op>[	COMM_PUSHMARK	-OP_***
//	-OP_*** 	]?[	COMM_OP+***	STAT_IFTRUE
//	STAT_IFTRUE	]:[	COMM_TOGGLESKIP	STAT_IFFALSE
//	STAT_IFFALSE	]	COMM_ENDSKIP	Final
//
//	Init    	#?$[	COMM_PUSHMARK	STAT_IFMATH
//	STAT_IFMATH 	]?[	COMM_IFMATH	STAT_IFTRUE
//	STAT_IFTRUE	]:[	COMM_TOGGLESKIP	STAT_IFFALSE
//	STAT_IFFALSE	]	COMM_ENDSKIP	Final
//
//	Init    	#>[	COMM_PUSHMARK	STAT_FLUSH
//	STAT_FLUSH	]	COMM_FLUSH	Final
//
//	Init    	#&[	COMM_PUSHMARK	STAT_GETVAR
//	STAT_GETVAR	]	COMM_LOADVAR	Final
//
//	Init     	#![	COMM_PUSHMARK   STAT_SETVAR
//	STAT_SETVAR	]=[	COMM_PUSHMARK	STAT_SETVARVAL
//	STAT_SETVARVAL	]	COMM_SAVEVAR	Final
//
//	Init    	#`[	COMM_PUSHMARK   STAT_EXECVAR
//	STAT_EXECVAR	]	COMM_EXECVAR	Final
//
//	Init    	#^[	COMM_PUSHMARK   STAT_EXEC
//	STAT_EXEC	]	COMM_EXEC	Final
//
//	Init     	#/[	COMM_PUSHMARK   STAT_REGEXPRSTR
//	STAT_REGEXPRSTR	]/[	COMM_PUSHMARK	STAT_REGEXPREXP
//	STAT_REGEXPREXP	]	COMM_REGEXPR	Final
//
//	Init     	#$[	COMM_PUSHMARK   STAT_MATH
//	STAT_MATH	]	COMM_CALCVAR	Final
//
//	Init    	#[	COMM_PUSHMARK   STAT_FIELDRANGE
//	STAT_FIELDRANGE	]	COMM_FIELDRANGE	Final
//
//	Init    	#.[	COMM_PUSHMARK   STAT_ARGVRANGE
//	STAT_ARGVRANGE	]	COMM_ARGVRANGE	Final
//
//	Init    	#|[	COMM_PUSHMARK   STAT_SETRECORD
//	STAT_SETRECORD	]	COMM_SETREC	Final
//
//	Init    	#:[	COMM_PUSHMARK   STAT_SAMEREC_RE
//	STAT_SAMEREC_RE	]	COMM_SAMEREC_RE	Final
//
//	Init    	#<[	COMM_PUSHMARK   STAT_PUSHFILE
//	STAT_PUSHFILE	]	COMM_PUSHFILE	Final
//
//	other    				Error
//

void setfield(int f,char *tbufpos) {
	*tbufpos = 0;
	if (ntf >= maxntf) error("Too many fields in outer template");
	tcomm[ntf++] = f;
	tfields[ntf] = tbufpos+1;
}

static void processtempl(const char *templfile) {
	if (ftempl) error("Attempting to open template file twice");
	if (!templfile) error("No template file name");
	if (templfile[0] != '-') {
		ftempl = fopen(templfile,"rt");
		if (!ftempl) error("Opening template file",templfile);
	} else ++templfile; //template from string!
	try {
		bool inhead = true;
		bool intail = false;
		int verbatim = 0;
		bodyid = 0;
		headid = tailid = -1;
		ntf = 0;
		stat.flush();
		tfields[0] = templbuf;
		int i;
		for (i = 0; i < templbuflen; ++i) {
			templbuf[i] = ftempl ? fgetc(ftempl) : getc_escape(templfile);
			if (ftempl ? feof(ftempl) : !templbuf[i]) break;
			if (verbatim > 0) {
				if (templbuf[i] == '#') {
					if (ftempl ? feof(ftempl) : !*templfile) error("EO outer template found while searching field id.");
					char c = ftempl ? fgetc(ftempl) : getc_escape(templfile);
					switch (c) {
					case '{':
						++verbatim;
						--i;
						continue;
					case '}':
						--verbatim;
						--i;
						continue;
					}
					if (i+1 < templbuflen) templbuf[++i] = c;
					continue;
				}
				continue;
			}
			if (templbuf[i] == ']') {
				switch (stat.currit()) {
				case STAT_IFCOND:
					switch (ftempl ? fgetc(ftempl) : getc_escape(templfile)) {
					case '=':
						stat.currit() = -OP_EQUAL;
						break;
					case '<':
						stat.currit() = -OP_SUBSTR;
						break;
					case '>':
						stat.currit() = -OP_SUPERSTR;
						break;
					case '^':
						stat.currit() = -OP_PREFIX;
						break;
					case '$':
						stat.currit() = -OP_POSTFIX;
						break;
					case '/':
						stat.currit() = -OP_MATCHES;
						break;
					default:
						error("Bad syntax in conditional expansion","unknown or missing <op>");
					}
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '[') error("Bad syntax in conditional expansion","]<op>[");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case STAT_IFTRUE:
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != ':') error("Bad syntax in conditional expansion","]:[");
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '[') error("Bad syntax in conditional expansion","]:[");
					stat.currit() = STAT_IFFALSE;
					setfield(COMM_TOGGLESKIP,templbuf+i);
					continue;
				case STAT_IFFALSE:
					stat.pop();
					setfield(COMM_ENDSKIP,templbuf+i);
					continue;
				case STAT_FLUSH:
					stat.pop();
					setfield(COMM_FLUSH,templbuf+i);
					continue;
				case STAT_GETVAR:
					stat.pop();
					setfield(COMM_LOADVAR,templbuf+i);
					continue;
				case STAT_SETVAR:
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '=') error("Bad syntax in conditional expansion","]=[");
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '[') error("Bad syntax in conditional expansion","]=[");
					stat.currit() = STAT_SETVARVAL;
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case STAT_SETVARVAL:
					stat.pop();
					setfield(COMM_SAVEVAR,templbuf+i);
					continue;
				case STAT_EXECVAR:
					stat.pop();
					setfield(COMM_EXECVAR,templbuf+i);
					continue;
				case STAT_EXEC:
					stat.pop();
					setfield(COMM_EXEC,templbuf+i);
					continue;
				case STAT_REGEXPRSTR:
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '/') error("Bad syntax in regexpr matching","]/[");
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '[') error("Bad syntax in regexpr matching","]/[");
					stat.currit() = STAT_REGEXPREXP;
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case STAT_REGEXPREXP:
					stat.pop();
					setfield(COMM_REGEXPR,templbuf+i);
					continue;
				case STAT_MATH:
					stat.pop();
					setfield(COMM_CALCVAR,templbuf+i);
					continue;
				case STAT_FIELDRANGE:
					stat.pop();
					setfield(COMM_FIELDRANGE,templbuf+i);
					continue;
				case STAT_ARGVRANGE:
					stat.pop();
					setfield(COMM_ARGVRANGE,templbuf+i);
					continue;
				case STAT_SETRECORD:
					stat.pop();
					setfield(COMM_SETREC,templbuf+i);
					continue;
				case STAT_SAMEREC_RE:
					stat.pop();
					setfield(COMM_SAMEREC_RE,templbuf+i);
					continue;
				case STAT_PUSHFILE:
					stat.pop();
					setfield(COMM_PUSHFILE,templbuf+i);
					continue;
				case -OP_EQUAL:
				case -OP_SUBSTR:
				case -OP_SUPERSTR:
				case -OP_PREFIX:
				case -OP_POSTFIX:
				case -OP_MATCHES:
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '?') error("Bad syntax in conditional expansion","]?[");
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '[') error("Bad syntax in conditional expansion","]?[");
					setfield(COMM_OP+stat.currit(),templbuf+i);
					stat.currit() = STAT_IFTRUE;
					continue;
				case STAT_IFMATH:
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '?') error("Bad syntax in math conditional expansion","]?[");
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '[') error("Bad syntax in math conditional expansion","]?[");
					setfield(COMM_IFMATH,templbuf+i);
					stat.currit() = STAT_IFTRUE;
					continue;
				}
				error("INTERNAL","invalid status value");
			}
			if (templbuf[i] == '#') {
				if (ftempl ? feof(ftempl) : !*templfile) error("EO outer template found while searching field id.");
				char c = ftempl ? fgetc(ftempl) : getc_escape(templfile);
				switch (c) {
				case '#': continue;
				case '{':
					++verbatim;
					--i;
					continue;
				case '}':
					error("Unmatched #} in outer template");
				case '\n':
				case '\r':
					--i;
					continue;
				case ']':
					templbuf[i] = ']';
					continue;
				case 'q':
					templbuf[i] = '"';
					continue;
				case 'r':
                                        if (i+orsep_len >= templbuflen) continue;
                                        strcpy(templbuf+i,orsep);
                                        i += orsep_len-1;
					continue;
				case 'f':
                                        if (i+ofsep_len >= templbuflen) continue;
                                        strcpy(templbuf+i,ofsep);
                                        i += ofsep_len-1;
					continue;
				case 't':
					{
						time_t t = time(NULL);
						const char *ct = ctime(&t);
						int l = strlen(ct+4)-1;
						if (i+l >= templbuflen) continue;
						strncpy(templbuf+i,ct+4,l);
						i += l-1;
						continue;
					}
				case '?':
					c = ftempl ? fgetc(ftempl) : getc_escape(templfile);
					if (c == '$') {
						stat.push(STAT_IFMATH);
						c = ftempl ? fgetc(ftempl) : getc_escape(templfile);
						if (c != '[') error("Bad syntax in math conditional expansion","#?$[");
					} else {
						stat.push(STAT_IFCOND);
						if (c != '[') error("Bad syntax in conditional expansion","#?[");
					}
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '>':
					stat.push(STAT_FLUSH);
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '[') error("Bad syntax in command flush","#>[");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '&':
					stat.push(STAT_GETVAR);
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '[') error("Bad syntax in variable get","#&[");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '!':
					stat.push(STAT_SETVAR);
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '[') error("Bad syntax in variable set","#![");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '`':
					stat.push(STAT_EXECVAR);
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '[') error("Bad syntax in variable exec","#`[");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '^':
					stat.push(STAT_EXEC);
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '[') error("Bad syntax in block exec","#^[");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '$':
					stat.push(STAT_MATH);
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '[') error("Bad syntax in variable calc","#$[");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '/':
					stat.push(STAT_REGEXPRSTR);
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '[') error("Bad syntax in regexpr matching","#/[");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '[':
					stat.push(STAT_FIELDRANGE);
					if (intail) error("Fields #[...] are not allowed in inline template tail");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '|':
					stat.push(STAT_SETRECORD);
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '[') error("Bad syntax in record set command","#|[");
					if (intail) error("Record set command not allowed in inline template tail");
					setfield(COMM_PUSHMARK,templbuf+i);
                                        continue;
				case ':':
					stat.push(STAT_SAMEREC_RE);
					if ((ftempl ? fgetc(ftempl) : getc_escape(templfile)) != '[') error("Bad syntax in same record regexpr command","#:[");
					if (intail) error("Same record regexpr command not allowed in inline template tail");
					setfield(COMM_PUSHMARK,templbuf+i);
                                        continue;
				case '<':
					if (intail) error("Set input file command not allowed in inline template tail");
					c = ftempl ? fgetc(ftempl) : getc_escape(templfile);
					switch (c) {
					case '[':
						stat.push(STAT_PUSHFILE);
						setfield(COMM_PUSHMARK,templbuf+i);
						break;
					case '-':
						setfield(COMM_POPFILE,templbuf+i);
						break;
					case '?':
						setfield(COMM_CHECKFILE,templbuf+i);
						break;
					case '0':
						setfield(COMM_SETINCHANNEL0,templbuf+i);
						break;
					case '1':
						setfield(COMM_SETINCHANNEL1,templbuf+i);
						break;
					case '2':
						setfield(COMM_SETINCHANNEL2,templbuf+i);
						break;
					case '3':
						setfield(COMM_SETINCHANNEL3,templbuf+i);
						break;
					case '4':
						setfield(COMM_SETINCHANNEL4,templbuf+i);
						break;
					case '5':
						setfield(COMM_SETINCHANNEL5,templbuf+i);
						break;
					case '6':
						setfield(COMM_SETINCHANNEL6,templbuf+i);
						break;
					case '7':
						setfield(COMM_SETINCHANNEL7,templbuf+i);
						break;
					case '8':
						setfield(COMM_SETINCHANNEL8,templbuf+i);
						break;
					case '9':
						setfield(COMM_SETINCHANNEL9,templbuf+i);
						break;
					default:
						error("Bad syntax in set input file command (expected a digit, '[', '-' or '?')","#<");
					}
                                        continue;
				case '.':
					{
						c = ftempl ? fgetc(ftempl) : getc_escape(templfile);
						if (c == '[') {
							stat.push(STAT_ARGVRANGE);
							setfield(COMM_PUSHMARK,templbuf+i);
							continue;
						}
						if (c < '0' || c > '9') error("Bad command line argument id. in template");
						int a = c-'0';
						if (a < _argc) {
							int l = strlen(_argv[a]);
							if (i+l < templbuflen) {
								strcpy(templbuf+i,_argv[a]);
								i += l-1;
								continue;
							}
						}
						--i;
						continue;
					}
				case '@':
					if (inhead) {
						inhead = false;
						setfield(COMM_NOP,templbuf+i);
						headid = 0;
						bodyid = ntf;
					} else if (intail) {
						error("Too many #@ in outer template");
					} else {
						intail = true;
						setfield(COMM_NOP,templbuf+i);
						tailid = ntf;
					}
					continue;
				case 'c':
					if (inhead) {
						--i;
						if (ftempl) fscanf(ftempl,"%d",&rrep);
						else rrep = strtol((char *)templfile,(char **)&templfile,10);
					} else error("#c can only be used in template head");
					continue;
				case 'n':
					if (inhead) {
						--i;
						if (ftempl) fscanf(ftempl,"%d",&nr);
						else nr = strtol((char *)templfile,(char **)&templfile,10);
					} else {
						setfield(COMM_NR,templbuf+i);
					}
					continue;
				case '+':
					setfield(COMM_NEXTREC,templbuf+i);
                                        continue;
				case '-':
					setfield(COMM_SAMEREC,templbuf+i);
                                        continue;
				case '_':
					setfield(COMM_BREAK,templbuf+i);
                                        continue;
				case '%':
					setfield(COMM_LOOP,templbuf+i);
                                        continue;
				case '*':
					if (intail) error("#* is not allowed in template tail");
					setfield(COMM_ALLFIELDS,templbuf+i);
					continue;
				case '1':
				case '2':
				case '3':
				case '4':
				case '5':
				case '6':
				case '7':
				case '8':
				case '9':
					if (intail) error("Fields #1,...,#9 are not allowed in template head");
					setfield(c-'0',templbuf+i);
					continue;
 				default: error("Bad field id. in template");
				}
			}
		}
		if (ftempl ? !feof(ftempl) : *templfile) error("Outer template too long");
		if (!stat.empty()) error("Unexpected end of file while parsing outer template");
		if (verbatim > 0) error("Unmatched #{ in outer template");
		templbuf[i] = 0;
	} catch (...) {
		if (ftempl) {
			fclose(ftempl);
			ftempl = 0;
		}
		throw;
	}
	if (ftempl) {
		fclose(ftempl);
		ftempl = 0;
	}
}


//: Inline template object

class intempl {
	enum {templbufleninit = 65536, templbuflenincr = 65536};
	enum {maxntfinit = 256, maxntfincr = 256};
	char *templbuf;
	int templbuflen;
	int headid;
	int bodyid;
	int tailid;
	int ntf;
	int maxntf;
	const char **tfields;
	int *tcomm;
	void clear() {
		if (tcomm) delete[] tcomm;
		tcomm = 0;
		if (tfields) delete[] tfields;
		tfields = 0;
		if (templbuf) delete[] templbuf;
		templbuf = 0;
		templbuflen = 0;
		ntf = 0;
		maxntf = 0;
	}
	void init() {
		headid = -1;
		bodyid = 0;
		tailid = -1;
		clear();
		templbuf = new char[templbufleninit+1];
		templbuf[0] = 0;
		templbuflen = templbufleninit;
		tfields = new const char *[maxntfinit+1];
		tcomm = new int[maxntfinit];
		maxntf = maxntfinit;
//printf("\nTemplate buffer has been allocated with %d bytes\n",templbuflen);fflush(stdout);
//printf("\nTemplate field array has been allocated with %d items\n",maxntf);fflush(stdout);
	}
	void shrink() {
		const char *laststr = tfields[ntf];
		int newlen = (laststr-templbuf)+strlen(laststr);
		char *newbuf = new char[newlen+1];
		memcpy(newbuf,templbuf,newlen+1);
		int *newcomm = new int[ntf];
		for (int i=0;i<ntf;++i) newcomm[i] = tcomm[i];
		const char **newf = new const char *[ntf+1];
		for (int i=0;i<=ntf;++i) newf[i] = (tfields[i]-templbuf)+newbuf;
		delete[] tcomm;
		tcomm = newcomm;
		delete[] tfields;
		tfields = newf;
		delete[] templbuf;
		templbuf = newbuf;
		templbuflen = newlen;
		maxntf = ntf;
//printf("\nTemplate buffer has shrunk down to %d bytes\n",templbuflen);fflush(stdout);
//printf("\nTemplate field array has shrunk to %d items\n",maxntf);fflush(stdout);
	}
	void growntf() {
		int *newcomm = new int[maxntf+maxntfincr];
		for (int i=0;i<ntf;++i) newcomm[i] = tcomm[i];
		const char **newf = new const char *[maxntf+maxntfincr+1];
		for (int i=0;i<=ntf;++i) newf[i] = tfields[i];
		delete[] tcomm;
		tcomm = newcomm;
		delete[] tfields;
		tfields = newf;
		maxntf += maxntfincr;
//printf("\nTemplate field array has grown to %d items\n",maxntf);fflush(stdout);
	}
	void growbuf(int l = templbuflenincr) {
		if (l < templbuflenincr) l = templbuflenincr;
		char *newbuf = new char[templbuflen+1+l];
		memcpy(newbuf,templbuf,templbuflen+1);
		for (int i=0;i<=ntf;++i) tfields[i] = (tfields[i]-templbuf)+newbuf;
		delete[] templbuf;
		templbuf = newbuf;
		templbuflen += l;
//printf("\nTemplate buffer has grown up to %d bytes\n",templbuflen);fflush(stdout);
	}
	void setfield(int f,char *tbufpos) {
		*tbufpos = 0;
		if (ntf >= maxntf) growntf(); //error("Too many fields in inline template");
		tcomm[ntf++] = f;
		tfields[ntf] = tbufpos+1;
	}
public:
	intempl() : templbuf(0), tfields(0), tcomm(0) {}
	~intempl() {clear();}
	void parse(const char *src);
	void vvprinthead(bool strict = false); // Never use it before calling parse()!!!
	void vvprintbody(bool strict = false); // Never use it before calling parse()!!!
	void vvprinttail(bool strict = false); // Never use it before calling parse()!!!
};

//: Inline Templates: Multiline definition
//.
//. An inline template can have a multiline definition as in the example:
//. #[named templ]#
//. #  HEAD#r#
//. # #@#
//. #  BODY1#r#
//. #  #?[#1]=[option]?[#
//. #    it is an option #2#
//. #  ]:[#
//. #    not an option #1#
//. #  ]#
//. #  #rBODY2#r#
//. # #@#
//. #  TAIL#r
//.

void intempl::vvprinthead(bool strict) {
	if (headid < 0) return;
	if (strict) {
		mark_stack.flush();
		skip_stack.flush();
	}
	skip_stack.push(skipping = -1);
	for (int tf = 0; tf < bodyid; ++tf) {
		vvprint(tfields[tf]);
		if (nf >= tcomm[tf]) vvprintcomm(tcomm[tf]);
	}
	skip_stack.pop();
	if (strict) {
		if (!skip_stack.empty()) error("INTERNAL","intempl::vvprinthead: Skip stack should be empty after template expansion");
		if (!mark_stack.empty()) error("INTERNAL","intempl::vvprinthead: Mark stack should be empty after template expansion");
	}
}

void intempl::vvprintbody(bool strict) {
	if (strict) {
		mark_stack.flush();
		skip_stack.flush();
	}
	skip_stack.push(skipping = -1);
	for (int tf = bodyid; tf < (tailid<0?ntf:tailid); ++tf) {
		vvprint(tfields[tf]);
		if (nf >= tcomm[tf]) vvprintcomm(tcomm[tf]);
	}
	if (tailid < 0) vvprint(tfields[ntf]);
	skip_stack.pop();
	if (strict) {
		if (!skip_stack.empty()) error("INTERNAL","intempl::vvprintbody: Skip stack should be empty after template expansion");
		if (!mark_stack.empty()) error("INTERNAL","intempl::vvprintbody: Mark stack should be empty after template expansion");
	}
}

void intempl::vvprinttail(bool strict) {
	if (tailid < 0) return;
	if (strict) {
		mark_stack.flush();
		skip_stack.flush();
	}
	skip_stack.push(skipping = -1);
	vvprint(tfields[tailid]);
	for (int tf = tailid; tf < ntf; ++tf) {
		if (nf >= tcomm[tf]) vvprintcomm(tcomm[tf]);
		vvprint(tfields[tf+1]);
	}
	skip_stack.pop();
	if (strict) {
		if (!skip_stack.empty()) error("INTERNAL","intempl::vvprinttail: Skip stack should be empty after template expansion");
		if (!mark_stack.empty()) error("INTERNAL","intempl::vvprinttail: Mark stack should be empty after template expansion");
	}
}

void intempl::parse(const char *src) {
	if (!src) error("INTERNAL: NULL string in inline template definition");
	try {
		init();
		bool inhead = true;
		bool intail = false;
		int verbatim = 0;
		bodyid = 0;
		headid = tailid = -1;
		ntf = 0;
		stat.flush();
		tfields[0] = templbuf;
		for (int i = 0; true; ++i) {
			if (i >= templbuflen) growbuf(i-templbuflen);
			templbuf[i] = getc_escape(src);
			if (!templbuf[i]) {
				if (!stat.empty()) error("Unexpected end of string while parsing an inline template");
				if (verbatim > 0) error("Unmatched #{ in inline template");
				shrink();
				return;
			}
			if (verbatim > 0) {
				if (templbuf[i] == '#') {
					if (!*src) {
						do
							if (!vvreadline() || linebuf[0] != '#') error("Unfinished multiline Inline Template");
						while (linebuf[1] == '#');
						--i;
						src=linebuf+1+strspn(linebuf+1," \t\v");
						continue;
					}
					char c = getc_escape(src);
					switch (c) {
					case '{':
						++verbatim;
						--i;
						continue;
					case '}':
						--verbatim;
						--i;
						continue;
					}
					if (++i >= templbuflen) growbuf(i-templbuflen);
					templbuf[i] = c;
					continue;
				}
				continue;
			}
			if (templbuf[i] == ']') {
				switch (stat.currit()) {
				case STAT_IFCOND:
					switch (getc_escape(src)) {
					case '=':
						stat.currit() = -OP_EQUAL;
						break;
					case '<':
						stat.currit() = -OP_SUBSTR;
						break;
					case '>':
						stat.currit() = -OP_SUPERSTR;
						break;
					case '^':
						stat.currit() = -OP_PREFIX;
						break;
					case '$':
						stat.currit() = -OP_POSTFIX;
						break;
					case '/':
						stat.currit() = -OP_MATCHES;
						break;
					default:
						error("Bad syntax in conditional expansion","unknown or missing <op>");
					}
					if (getc_escape(src) != '[') error("Bad syntax in conditional expansion","]<op>[");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case STAT_IFTRUE:
					if (getc_escape(src) != ':') error("Bad syntax in conditional expansion","]:[");
					if (getc_escape(src) != '[') error("Bad syntax in conditional expansion","]:[");
					stat.currit() = STAT_IFFALSE;
					setfield(COMM_TOGGLESKIP,templbuf+i);
					continue;
				case STAT_IFFALSE:
					stat.pop();
					setfield(COMM_ENDSKIP,templbuf+i);
					continue;
				case STAT_FLUSH:
					stat.pop();
					setfield(COMM_FLUSH,templbuf+i);
					continue;
				case STAT_GETVAR:
					stat.pop();
					setfield(COMM_LOADVAR,templbuf+i);
					continue;
				case STAT_SETVAR:
					if (getc_escape(src) != '=') error("Bad syntax in variable set","]=[");
					if (getc_escape(src) != '[') error("Bad syntax in variable set","]=[");
					stat.currit() = STAT_SETVARVAL;
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case STAT_SETVARVAL:
					stat.pop();
					setfield(COMM_SAVEVAR,templbuf+i);
					continue;
				case STAT_EXECVAR:
					stat.pop();
					setfield(COMM_EXECVAR,templbuf+i);
					continue;
				case STAT_EXEC:
					stat.pop();
					setfield(COMM_EXEC,templbuf+i);
					continue;
				case STAT_REGEXPRSTR:
					if (getc_escape(src) != '/') error("Bad syntax in regexpr matching","]/[");
					if (getc_escape(src) != '[') error("Bad syntax in regexpr matching","]/[");
					stat.currit() = STAT_REGEXPREXP;
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case STAT_REGEXPREXP:
					stat.pop();
					setfield(COMM_REGEXPR,templbuf+i);
					continue;
				case STAT_MATH:
					stat.pop();
					setfield(COMM_CALCVAR,templbuf+i);
					continue;
				case STAT_FIELDRANGE:
					stat.pop();
					setfield(COMM_FIELDRANGE,templbuf+i);
					continue;
				case STAT_ARGVRANGE:
					stat.pop();
					setfield(COMM_ARGVRANGE,templbuf+i);
					continue;
				case STAT_SETRECORD:
					stat.pop();
					setfield(COMM_SETREC,templbuf+i);
					continue;
				case STAT_SAMEREC_RE:
					stat.pop();
					setfield(COMM_SAMEREC_RE,templbuf+i);
					continue;
				case STAT_PUSHFILE:
					stat.pop();
					setfield(COMM_PUSHFILE,templbuf+i);
					continue;
				case -OP_EQUAL:
				case -OP_SUBSTR:
				case -OP_SUPERSTR:
				case -OP_PREFIX:
				case -OP_POSTFIX:
				case -OP_MATCHES:
					if (getc_escape(src) != '?') error("Bad syntax in conditional expansion","]?[");
					if (getc_escape(src) != '[') error("Bad syntax in conditional expansion","]?[");
					setfield(COMM_OP+stat.currit(),templbuf+i);
					stat.currit() = STAT_IFTRUE;
					continue;
				case STAT_IFMATH:
					if (getc_escape(src) != '?') error("Bad syntax in math conditional expansion","]?[");
					if (getc_escape(src) != '[') error("Bad syntax in math conditional expansion","]?[");
					setfield(COMM_IFMATH,templbuf+i);
					stat.currit() = STAT_IFTRUE;
					continue;
				}
				error("INTERNAL","invalid status value");
			}
			if (templbuf[i] == '#') {
				if (!*src) {
					do
						if (!vvreadline() || linebuf[0] != '#') error("Unfinished multiline Inline Template");
					while (linebuf[1] == '#');
					--i;
					src=linebuf+1+strspn(linebuf+1," \t\v");
					continue;
				}
				char c = getc_escape(src);
				switch (c) {
				case '#': continue;
				case '{':
					++verbatim;
					--i;
					continue;
				case '}':
					error("Unmatched #} in inline template");
				case '\n':
					--i;
					continue;
				case ']':
					templbuf[i] = ']';
					continue;
				case 'q':
					templbuf[i] = '"';
					continue;
				case 'r':
                                        if (i+orsep_len >= templbuflen) growbuf(orsep_len);
                                        strcpy(templbuf+i,orsep);
                                        i += orsep_len-1;
					continue;
				case 'f':
                                        if (i+ofsep_len >= templbuflen) growbuf(ofsep_len);
                                        strcpy(templbuf+i,ofsep);
                                        i += ofsep_len-1;
					continue;
				case 't':
                                        {
						time_t t = time(NULL);
						const char *ct = ctime(&t);
						int l = strlen(ct+4)-1;
						if (i+l >= templbuflen) growbuf(l);
						strncpy(templbuf+i,ct+4,l);
						i += l-1;
						continue;
					}
				case '?':
					c = getc_escape(src);
					if (c == '$') {
						stat.push(STAT_IFMATH);
						c = getc_escape(src);
						if (c != '[') error("Bad syntax in math conditional expansion","#?$[");
					} else {
						stat.push(STAT_IFCOND);
						if (c != '[') error("Bad syntax in conditional expansion","#?[");
					}
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '>':
					stat.push(STAT_FLUSH);
					if (getc_escape(src) != '[') error("Bad syntax in command flush","#>[");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '&':
					stat.push(STAT_GETVAR);
					if (getc_escape(src) != '[') error("Bad syntax in variable get","#&[");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '!':
					stat.push(STAT_SETVAR);
					if (getc_escape(src) != '[') error("Bad syntax in variable set","#![");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '`':
					stat.push(STAT_EXECVAR);
					if (getc_escape(src) != '[') error("Bad syntax in variable exec","#`[");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '^':
					stat.push(STAT_EXEC);
					if (getc_escape(src) != '[') error("Bad syntax in block exec","#^[");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '$':
					stat.push(STAT_MATH);
					if (getc_escape(src) != '[') error("Bad syntax in variable calc","#$[");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '/':
					stat.push(STAT_REGEXPRSTR);
					if (getc_escape(src) != '[') error("Bad syntax in regexpr matching","#/[");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '[':
					stat.push(STAT_FIELDRANGE);
					if (intail) error("Fields #[...] are not allowed in inline template tail");
					setfield(COMM_PUSHMARK,templbuf+i);
					continue;
				case '|':
					stat.push(STAT_SETRECORD);
					if (getc_escape(src) != '[') error("Bad syntax in record set command","#|[");
					if (intail) error("Record set command not allowed in inline template tail");
					setfield(COMM_PUSHMARK,templbuf+i);
                                        continue;
				case ':':
					stat.push(STAT_SAMEREC_RE);
					if (getc_escape(src) != '[') error("Bad syntax in same record regexpr command","#:[");
					if (intail) error("Same record regexpr command not allowed in inline template tail");
					setfield(COMM_PUSHMARK,templbuf+i);
                                        continue;
				case '<':
					if (intail) error("Set input file command not allowed in inline template tail");
					c = getc_escape(src);
					switch (c) {
					case '[':
						stat.push(STAT_PUSHFILE);
						setfield(COMM_PUSHMARK,templbuf+i);
						break;
					case '-':
						setfield(COMM_POPFILE,templbuf+i);
						break;
					case '?':
						setfield(COMM_CHECKFILE,templbuf+i);
						break;
					case '0':
						setfield(COMM_SETINCHANNEL0,templbuf+i);
						break;
					case '1':
						setfield(COMM_SETINCHANNEL1,templbuf+i);
						break;
					case '2':
						setfield(COMM_SETINCHANNEL2,templbuf+i);
						break;
					case '3':
						setfield(COMM_SETINCHANNEL3,templbuf+i);
						break;
					case '4':
						setfield(COMM_SETINCHANNEL4,templbuf+i);
						break;
					case '5':
						setfield(COMM_SETINCHANNEL5,templbuf+i);
						break;
					case '6':
						setfield(COMM_SETINCHANNEL6,templbuf+i);
						break;
					case '7':
						setfield(COMM_SETINCHANNEL7,templbuf+i);
						break;
					case '8':
						setfield(COMM_SETINCHANNEL8,templbuf+i);
						break;
					case '9':
						setfield(COMM_SETINCHANNEL9,templbuf+i);
						break;
					default:
						error("Bad syntax in set input file command (expected a digit, '[', '-' or '?')","#<");
					}
                                        continue;
				case '.':
					{
						c = getc_escape(src);
						if (c == '[') {
							stat.push(STAT_ARGVRANGE);
							setfield(COMM_PUSHMARK,templbuf+i);
							continue;
						}
						if (c < '0' || c > '9') error("Bad command line argument id. in inline template");
						int a = c-'0';
						if (a < _argc) {
							int l = strlen(_argv[a]);
							if (i+l >= templbuflen) growbuf(l);
							strcpy(templbuf+i,_argv[a]);
							i += l-1;
						} else --i;
						continue;
					}
				case '@':
					if (inhead) {
						inhead = false;
						setfield(COMM_NOP,templbuf+i);
						headid = 0;
						bodyid = ntf;
					} else if (intail) {
						error("Too many #@ in inline template");
					} else {
						intail = true;
						setfield(COMM_NOP,templbuf+i);
						tailid = ntf;
					}
					continue;
				case 'c': error("#c cannot be used in inline template");
				case 'n':
					setfield(COMM_NR,templbuf+i);
					continue;
				case '+':
					setfield(COMM_NEXTREC,templbuf+i);
                                        continue;
				case '-':
					setfield(COMM_SAMEREC,templbuf+i);
                                        continue;
				case '_':
					setfield(COMM_BREAK,templbuf+i);
                                        continue;
				case '%':
					setfield(COMM_LOOP,templbuf+i);
                                        continue;
				case '*':
					if (intail) error("#* is not allowed in inline template tail");
					setfield(COMM_ALLFIELDS,templbuf+i);
					continue;
				case '1':
				case '2':
				case '3':
				case '4':
				case '5':
				case '6':
				case '7':
				case '8':
				case '9':
					if (intail) error("Fields #1,...,#9 are not allowed in inline template tail");
					setfield(c-'0',templbuf+i);
					continue;
				default:
					error("Bad field id. in inline template");
				}
			}
		}
	} catch (...) {
		throw;
	}
}

//: Inline Templates: Nested usage
//.
//. There are two tables of inline templates: one named and one unnamed
//. The unnamed template table is a stack
//. There is also a stack of pointers to active inline templates
//. #[name]...       creates or modifies and sets a named inline template
//. #[-name]...      creates or modifies a named inline template
//. #[name]          sets an already existing named inline template (or error)
//. #[]              closes current named inline template (or error)
//. #[[...           creates or modifies and sets an unnamed template
//. #[[              sets an already existing unnamed template (or error)
//. #]]              closes current named template if there is one 
//.                   and closes an unnamed template (or error)
//. At end of data, all the active templates are closed sequentially


//: Active inline template stack

DynStack<intempl *> activeintempls("Active Template Stack",32,32);

inline intempl *currit() {return activeintempls.empty()? 0 : activeintempls.currit();}


//: Named inline template table

class NamedTempl : public BasicId {
	intempl *pt;
public:
	NamedTempl(const char *name,bool own = false) : BasicId(name,own), pt(0) {
//printf("\nCreating NamedTempl object associated to \"%s\"\n",operator const char *());fflush(stdout);
	}
	~NamedTempl() {
//printf("\nDestroying NamedTempl object associated to \"%s\"\n",operator const char *());fflush(stdout);
		if (pt) delete pt; pt = 0;
	}
	void fill(intempl *_pt) { // Object *_pt is always owned (and finally deleted) by the NamedTempl object!!! 
//printf("\nTransferring Templ object associated to \"%s\"\n",operator const char *());fflush(stdout);
if (pt) error("Attempting to overwrite the intempl pointer in a NamedTempl object");
		if (pt) error("Attempting to set twice a NamedTempl object");
		pt = _pt;
	}	 
	void parse(const char *src) {pt->parse(src);} // Never call it before using fill()!!!
	void vvprinthead(bool strict = false) {pt->vvprinthead(strict);} // Never call it before using fill()!!!
	void vvprintbody(bool strict = false) {pt->vvprintbody(strict);} // Never call it before using fill()!!!
	void vvprinttail(bool strict = false) {pt->vvprinttail(strict);} // Never call it before using fill()!!!
	intempl *getobjpointer() {return pt;}
};

HashTable<NamedTempl,BasicId> ittable(256);


//: Unnamed inline template stack

DynStack<intempl> unnamedintempls("Unnamed Template Stack",32,0); // Do not make it growable: it can invalidate existing references


void templup() {
	unnamedintempls.push();
	activeintempls.push(&unnamedintempls.currit());
}

void templdown() {
	activeintempls.pop();
	unnamedintempls.pop();
}

void filltempl(const char *t) {
	currit()->parse(t);
}

//: Direct (possibly nested) inline template expansion (for direct templates, filenames, etc.)

//DynStack<intempl> directtempls("Direct Template Stack",32,0);

#ifdef REPORT
int expcounter = 0;
#endif

static void expanddirecttemplate(const char *p) {
#ifdef REPORT
	++expcounter;
#endif
//	directtempls.push();
//	intempl &auxtempl = directtempls.currit(); // CAUTION!!!: DynStack can reallocate objects!
	intempl *pauxtempl = 0;
	try {
		intempl &auxtempl = *(pauxtempl = new intempl);
		auxtempl.parse(p);
		skip_stack.push(skipping = -1);
	#ifdef	NO_LOCAL_LOOPS_AND_HALTS_IN_DIRECT_TEMPLATES
		auxtempl.vvprinthead();
		auxtempl.vvprintbody();
		auxtempl.vvprinttail();
	#else
		enable = 1;
		auxtempl.vvprinthead();
		do {
			loop = 0;
			enable = 1;
			auxtempl.vvprintbody();
		} while (loop);
		enable = 1;
		auxtempl.vvprinttail();
		enable = 1;
	#endif
		skip_stack.pop();
		delete pauxtempl;
	} catch (...) {
		if (pauxtempl) delete pauxtempl;
		throw;
	}
//	directtempls.pop();
}

static void processspecial() {
	enable = 1;
	switch (linebuf[1]) {
	case '<': 
		vvclearfields();
		expanddirecttemplate(linebuf+2);
		if (!exppos) error("No input file specified in #<<filename>");
		expbuf[exppos]=0;
		push_file(expbuf);
		exppos=0;
		break;
	case '!':
		vvclearfields();
		expanddirecttemplate(linebuf+2);
		vvflush();
		break;
	case '[': 
		switch (linebuf[2]) {
		case '[':
			templup();
			if (linebuf[3]) filltempl(linebuf+3);
			currit()->vvprinthead();
			break;
		case ']':
			if (activeintempls.empty()) error("Active inline template stack underflow");
			currit()->vvprinttail();
			activeintempls.pop();
			break;
		default:
			{
				char *name = linebuf+2;
				bool onlydef = false;
				if (name[0] == '-') {
					++name;
					onlydef = true;
				}
				char *p = strchr(name,']');
				if (!p) error("Bad named inline template syntax","Missing delimiter ']'");
				if (p == name) error("Bad named inline template syntax","Empty template name");
				*p = 0;
				if (!onlydef && !p[1]) {
					NamedTempl *pt = ittable.find(name);
					if (!pt) error("Unknown named template");
					activeintempls.push(pt->getobjpointer()); 
					pt->vvprinthead();
				} else {
					NamedTempl *pt = ittable.insert(name,true); // Duplicate ID buffer on actual insertion
					if (!pt->getobjpointer()) {
						pt->fill(new intempl); // Object is always owned by *pt
					}
					pt->parse(p+1);
					if (!onlydef) {
						activeintempls.push(pt->getobjpointer());
						pt->vvprinthead();
					}
				}
			}
		}
		break;
	case ']':
		if (linebuf[2] == ']') {
			if (activeintempls.empty()) error("Active inline template stack underflow");
			currit()->vvprinttail();
			templdown();
		} else error("Bad inline template syntax","Missing second ']' in #]]");
		break;
	}
	vvflush();
}

static void vvprinthead(bool strict=true) {
	enable = 1;
	if (headid < 0) return;
	if (strict) {
		mark_stack.flush();
		skip_stack.flush();
	}
	skip_stack.push(skipping = -1);
	for (int tf = 0; tf < bodyid; ++tf) {
		vvprint(tfields[tf]);
		if (nf >= tcomm[tf]) vvprintcomm(tcomm[tf]);
	}
	skip_stack.pop();
	if (strict) {
		if (!skip_stack.empty()) error("INTERNAL","::vvprinthead: Skip stack should be empty after template expansion");
		if (!mark_stack.empty()) error("INTERNAL","::vvprinthead: Mark stack should be empty after template expansion");
	}
	vvflush();
}

static void vvprintbody(bool strict=true) {
	do {
		loop = 0;
		enable = 1;
		++nr;
		if (currit()) {
			currit()->vvprintbody(strict);
		} else {
			if (strict) {
				mark_stack.flush();
				skip_stack.flush();
			}
			skip_stack.push(skipping = -1);
			for (int tf = bodyid; tf < (tailid<0?ntf:tailid); ++tf) {
				vvprint(tfields[tf]);
				if (nf >= tcomm[tf]) vvprintcomm(tcomm[tf]);
			}
			if (tailid < 0) vvprint(tfields[ntf]);
			skip_stack.pop();
			if (strict) {
				if (!skip_stack.empty()) error("INTERNAL","::vvprintbody: Skip stack should be empty after template expansion");
				if (!mark_stack.empty()) error("INTERNAL","::vvprintbody: Mark stack should be empty after template expansion");
			}
		}
		vvflush();
	} while (loop);
	loop = 0;
}

static void vvprinttail(bool strict=true) {
	while (!activeintempls.empty()) {
		enable = 1;
		currit()->vvprinttail(strict);
		activeintempls.pop();
	}
	vvflush();
	enable = 1;
	if (tailid < 0) return;
	if (strict) {
		mark_stack.flush();
		skip_stack.flush();
	}
	skip_stack.push(skipping = -1);
	vvprint(tfields[tailid]);
	for (int tf = tailid; tf < ntf; ++tf) {
		if (nf >= tcomm[tf]) vvprintcomm(tcomm[tf]);
		vvprint(tfields[tf+1]);
	}
	skip_stack.pop();
	if (strict) {
		if (!skip_stack.empty()) error("INTERNAL","::vvprinttail: Skip stack should be empty after template expansion");
		if (!mark_stack.empty()) error("INTERNAL","::vvprinttail: Mark stack should be empty after template expansion");
	}
	vvflush();
}

const char *helpinfo[] = {
	" Use - to indicate that data comes from stdin.",
	" The usual escape chars are interpreted in <field_sep> and <output_rec_sep>.",
	" Start with - to specify the template directly in the command line.",
	"",
	"TEMPLATE FILE FORMAT:",
	" templ_file contents is [<header>#@]<body>[#@<tail>]",
	"  #1 ... #9 indicates field substitution in <body>.",
	"  #.0 ... #.9 inserts the command line argument $0 ... $9.",
	"  #* indicates substitution by all fields in <body>.",
	"  #n inserts record number.",
	"  #n<number> (only in header) initializes record counter.",
	"  #c<number> (only in header) initializes replication counter.",
	"  ## inserts # character.",
	"  #] inserts ] character.",
	"  #q inserts '\"' character.",
	"  #r inserts output record separator (def. is '\\n').",
	"  #f inserts field separator (def. is ',').",
	"  #t inserts time stamp as in ctime().",
	"  #<CR> is ignored.",
	"",
	"CONDITIONAL EXPANSION:",
	" The syntax #?[<expr1>]<op>[<expr2>]?[<trueexpr>]:[<falseexpr>] is used for",
	"  conditional expansion: <expr1> is compared to <expr2> with operator <op>.",
	"  Depending on the result, either <trueexpr> or <falseexpr> is expanded.",
	"  <op> is one of '=', '<', '>', '^', '$' or '/', meaning equality, substring,",
	"  superstring, prefix, postfix, or regular expression matching. In the latter",
	"  case, <expr1> is the string and <expr2> is a grep-style regular expression.",
	" Conditional expressions can be nested in any possible way.",
	"",
	"REGULAR EXPRESSION MATCHING:",
	" In addition to the regular expression matching test provided as an operator",
	"  in the conditional expansion command, the following command syntax:",
	"   #/[<string>]/[<regexpr>]",
	"  expands to the first longest substring of <string> matching <regexpr>, a",
	"  grep-like regular expression. If there is no possible match, the expansion",
	"  results in the empty string. Therefore, there is no distintion between this",
	"  case, and the case of a matching empty string (like in the regular expression",
	"  \"$\". Use the conditional expansion construction to tell apart the two cases.",
	" Most of grep extended regular expression functionalities are supported,",
	"  including repetition operators and back references.",
	" As an extension, a '/' in <regexpr> splits it into the proper regular expression",
	"  and a template, with the syntax:",
	"  #/[<string>]/[<regexpr>/<template>]",
	" The template specifies the expansion result, and it can contain both back",
	"  references and normal text. Any literal '/' must be escaped as \"\\/\".",
	"  Also '\\\\', '\\t' and '\\' are interpreted in the usual way. Improper or",
	"  unmatched back references in the template are just ignored.",
	" Any template string in a <regexpr> operand of the conditional expansion regular",
	" expression matching command will be ignored. As well, any character following a",
	"  second '/', including itself, in the <regexpr> argument of #/ is ignored, with",
	"  one exception. Namely, the syntax:",
	"   #/[<string>]/[<regexpr>/<template>/|]",
	"  that is, with the character '|', identifies the back references with input",
	"  fields, where #1 is the whole matching substring, #2 is the back reference \\1,",
	"  and #* is the usual concatenation of input fields and output field separators",
	"  #1#f#2#f#3...",
	" As soon as a field updating command like #+, #- or #| is called, the previous",
	"  association is lost. The same occurs after calling any regexpr operator, with",
	"  or without the '|' character.",
	"",
	"SPECIAL RECORDS:",
	" Data records starting by '#' are ignored except for those starting",
	"  with '#[', '#]', '#!' or '#<', that are special data records",
	"",
	"INLINE TEMPLATES:",
	" '#[[<aux templ>' defines an inline template to be used with subsequent records.",
	" In inline templates, escape chars are also interpreted.",
	" '#[[' starts using a previously defined inline template.",
	"  while '#]]' stops using the current inline template.",
	" #c cannot be used in the header of an inline template.",
	" In an inline template #n can be used in the header but with the same meaning as",
	"  in the body.",
	" #<ret> breaks the inline template definition into multiple lines.",
	" Additional lines start with '#' and whitespaces following it are ignored.",
	" Use of #1,...#9,#*,#+ and conditional expansion is allowed in all parts of an",
	"  inline template.",
	"",
	"INLINE NAMED TEMPLATES:",
	" Inline templates can be referred by name, if the syntax",
	"  '#[<name>]<aux templ>' is used.",
	" A previously defined template is activated by using '#[<name>]'",
	"  and its use is stopped with '#[]'.",
	" '#[-<name>]<aux templ>' defines but doesn't activate the template.",
	" '#[-<name>]' sets the named template to the empty template, but doesn't",
	"  activate it.",
	" The use of '#[[' and '#]]' and '#[<name>]' and '#[]' can be nested.",
	"",
	"VARIABLES IN TEMPLATES:",
	" Text variables can be defined and used in templates.",
	" #![<name>]=[<value>] sets <name> to <value>.",
	" #&[<name>] expands to the value associated to <name>.",
	" #`[<name>] expands to the value associated to <name> interpreted as a direct",
	"  template.",
	" #^[<templ>] directly uses the direct template specified in <templ>.",
	" Names and values can both use conditional expansion and recursion.",
	" Variable access is also allowed in template head and tail.",
	" The special variable names __ORSEP__, __IFSEP__ and __OFSEP__ are used to get",
	"  or modify resp. the values of the output record, input field and output field",
	"  separators.",
	" The special variable names __NR__ and __RREP__ are used to get or modify resp.",
	"  the values of the record counter (like #n) and the input record replication",
	"  counter (like #c).",
	" The special read-only variable name __NF__ is used to get but not modify the",
	"  number of fields in the currently processed record.",
	" Moreover, the obvious read-only variable __VERSION__ also exists.",
	"",
	"INLINE DIRECT TEMPLATES:",
	" '#!<aux templ>' defines an inline template and executes it",
	"  on a single empty data record. Use it for variable initialization.",
	"",
	"MATHEMATICAL EXPRESSIONS:",
	" #$[<expr>] expands to the numeric value computed from the string <expr>.",
	" #$[&<name>] expands to the numeric value computed from the string associated to",
	"  variable <name>.",
	" #$[`<name>] expands to the numeric value computed from the string resulting",
	"  from the interpretation of the value of <name> as a direct template.",
	" The mathematical expression can include variable assignments, that are kept for",
	"  further evaluations.",
	" Examples are 'x=5', 'y=x=sin(pi)', 'x+=3'.",
	" The commands #$[@<expr>], #$[@&<expr>], #$[@`<expr>] evaluate the expression",
	"  but produce no output.",
	" A format specification can be appended to the mathematical expression as in",
	"  #$[@<expr>;%.7g].",
	" The format specification is separated from the expression by a ';'.",
	" Format syntax follows printf rules, with some exceptions.",
	"",
	"MATH CONDITIONAL EXPANSION:",
	" #?$[<expr>]?[<trueexpr>]:[<falseexpr>] expands <trueexpr> if <expr>, as a",
	"  mathematical expression, evaluates to a nonzero value, and it expands",
	"  <falseexpr> otherwise.",
	"  <expr> can take any of the forms defined before.",
	"",
	"SPECIAL RECORD MANAGEMENT FEATURES:",
	" #+ reads next record and continues template expansion.",
	" #- rescans again the input fields, perhaps with a different input field",
	"  separator.",
	" #:[<regexpr>] rescans again the input fields, but using as separators the",
	"  matching substrings with <regexpr>. The separators are also stored as input",
	"  fields (even fields #2, #4... are filled with the separators and odd fields #1,"
	"  #3... are the proper fields).",
	" #|[<foo>] replaces current record by <foo>, rescans input fields and continue",
	"  processing.",
	"",
	"EXTENDED COMPACT FIELD SEQUENCE SPECIFICATION:",
	" #[<sequence>] produces a sequence of input fields separated by the output",
	"  field separator.",
	" The sequence specification syntax is a comma-separated sequence of ranges.",
	"  x:y:z specifies a sequence of fields from x to z with (nonzero) increment y.",
	"  z is only in the range when there exists a nonnegative integer t such that",
	"  z=x+ty. The range is empty if z is different form x but z-x and y have",
	"  different signs.",
	"  x:y is an abbreviation of x:1:y.",
	"  x specifies an individual field (it is an abbreviation of x:1:x).",
	"  0 means the last field, -1 the second last one, etc.",
	" In a similar way, you can use #.[<sequence>] for command line arguments, but",
	"  now 0 means",
	"  argv[0], -1 means argv[argc-1], -2 means argv[argc-2], etc.",
	"",
	"CHARACTER BASED PROCESSING:",
	" An empty input field separator causes the input fields to be defined as the",
	"  individual characters in the input record.",
	"",
	"NESTED DATA FILES:",
	" Use '#<<filename>' to insert a data file.",
	" '#<-' reads from stdin.",
	" <filename> can use macro expansions (conditional expansions, variables,...).",
	"",
	"MULTIPLE INPUT CHANNELS:",
	" 10 independent input channels (each supporting nested input files) are",
	"  implemented.",
	" They are accesible via the following template commands:",
	"  #<0,...,#<9 select the corresponding input channel. The default initial",
	"   channel is 0.",
	"  #<? checks whether the current channel is ready or exhausted (it expands to",
	"   0 at EOF, and to 1 otherwise).",
	"  #<- closes the current file in the current channel.",
	"  #<[filename] pushes file named filename into the input file stack associated",
	"   to the current input channel.",
	"",
	"IMMEDIATE OUTPUT:",
	" #>[<text>] immwdiately writes <text> into stdout. Macro expansions in <text>",
	"  are previously processed.",
	"",
	"EXPANSION CONTROL COMMANDS:",
	" #_ breaks the expansion of the head, of the body or of the tail of the outer",
	"  template or any direct template.",
	" #% breaks the expansion when it occurs in the body of the outer template or",
	"  any direct template, and repeats the expansion of the body again (possibly",
	"  entering into an enless loop). Use it with care!",
	" #{, #} can be used to surround delayed expansion code. #-commands inside the",
	"  text delimited by these commands are not interpreted. Nested #{...#} groups",
	"  are supported, and the inner #{ #} are kept unchanged. However, escape chars",
	"  are still interpreted. Multiline direct templates are supported as before:",
	"  Lines ending in a single # followed by a line starting with a single # are",
	"  joined (removing the two #, the newline character and any subsequent white",
	"  spaces).",
#ifdef	NO_LOCAL_LOOPS_AND_HALTS_IN_DIRECT_TEMPLATES
	"*** This compilation disabled halting and looping within direct templates! ***",
#endif
	NULL
};

void givehelp() {
	fprintf(stderr,"LINEPROCX version %d.%d (%s)\n",VER_MAJOR,VER_MINOR,DATE);
	fprintf(stderr,"\nUsage: %s <template_file> <data_file> [<field_sep> [<output_rec_sep>]]\n",_argv[0]);
	for (const char **p=helpinfo;*p;p++) fprintf(stderr,"%s\n",*p);
}

int main(int argc, char **argv) {
	_argc = argc;
        _argv = argv;
	if (argc < 3) {
		givehelp();
		exit(0);
	}
	NamedVar *pifsep_var = insertsysvar("__IFSEP__",ifsep,maxseplen);
	NamedVar *pofsep_var = insertsysvar("__OFSEP__",ofsep,maxseplen);
	NamedVar *porsep_var = insertsysvar("__ORSEP__",orsep,maxseplen);
	insertsysvar("__RREP__",rrep);
	insertsysvar("__NR__",nr);
	insertsysvar_ro("__NF__",nf);
	insertsysvar_ro("__INPUT__",in_channel);
	insertsysvar_ro("__VERSION__",version);
	if (argc >= 4 /*&& argv[3][0]*/) {
		pifsep_var->setesc(argv[3]);
		pofsep_var->setesc(argv[3]);
	}
	if (argc >= 5 /*&& argv[4][0]*/) 
		porsep_var->setesc(argv[4]);
	try {
		processtempl(argv[1]);
		push_file(argv[2]);
		vvprinthead();
		while (vvreadline()) {
			//if (!vvreadline()) continue;
			if (linebuf[0]=='#') {processspecial(); continue;}
			vvgetfields();
			for (int i = 0; i < rrep; ++i) vvprintbody();
		}
		vvprinttail();
	} catch (x _x) {
		fprintf(stderr,"\nERROR: %s",_x.m1 ? _x.m1 : "<Not specified>");
		if (_x.m2) fprintf(stderr,": %s",_x.m2);
		fprintf(stderr,"\n");
		vvclose();
		exit(0);
	} catch (x__VExpr _x) {
		fprintf(stderr,"\nERROR (VEXPR): %s",_x.m1 ? _x.m1 : "<Not specified>");
		if (_x.m2) fprintf(stderr,": %s",_x.m2);
		fprintf(stderr,"\n");
		vvclose();
		exit(0);
	} catch (x_underflow _x) {
		fprintf(stderr,"\nUncatched stack underflow: %s\n",_x.id?_x.id:"");
		vvclose();
		exit(0);
	} catch (x_overflow _x) {
		fprintf(stderr,"\nUncatched stack overflow: %s\n",_x.id?_x.id:"");
		vvclose();
		exit(0);
	} catch (x_badindex _x) {
		fprintf(stderr,"\nBad stack index: %s\n",_x.id?_x.id:"");
		vvclose();
		exit(0);
	} catch (...) {
		fprintf(stderr,"\nUnknown exception thrown\n");
		vvclose();
		exit(0);
	}
	vvclose();
#ifdef REPORT
	fprintf(stderr,"\nUsage statistics report:");
	fprintf(stderr,"\n   Number of direct templates expansions: %d",expcounter);
	fprintf(stderr,"\n   Collisions in named tamplates table: %.2f%%",100*ittable.avcoll());
	fprintf(stderr,"\n   Collisions in variables table: %.2f%%",100*vartable.avcoll());
	fprintf(stderr,"\n");
#endif
	return 0;
}