From 369c346b93b0327f9e0f63849aa635113882af75 Mon Sep 17 00:00:00 2001 From: Vladimir Glazounov Date: Thu, 18 Jan 2007 08:44:15 +0000 Subject: INTEGRATION: CWS dmake47 (1.10.2); FILE MERGED 2006/12/22 04:36:58 vq 1.10.2.10: #i61856# Remove another verbose debugging statement. 2006/12/21 03:16:17 vq 1.10.2.9: #i61856# Child process handling improvements. 2006/12/02 19:10:55 vq 1.10.2.8: #i72210# Add missing detail and a testcase. 2006/12/02 18:25:49 vq 1.10.2.7: #i72210# Teach spawn enabled dmake to honor the '-' recipe switch. 2006/12/01 01:16:58 vq 1.10.2.6: #i61856# Fix "Internal Error: Child is missing .." problem. 2006/11/27 22:23:14 vq 1.10.2.5: #i61856# Improve (dmake internal) process handling. 2006/11/19 05:27:24 vq 1.10.2.4: #i71704# Let the global .SEQUENTIAL attribute implicitely set MAXPROCESS=1 and disallow MAXPROCESS to be changed if the global .SEQUENTIAL is set. 2006/11/17 21:56:23 vq 1.10.2.3: #i61856# Implement better way to wait for a process queue to finish without obstructing other process queues during parallel makefile processing. 2006/11/15 19:43:12 vq 1.10.2.2: #i71582# Make dmake usable on OpenBSD. (Patch separated from CWS openbsd01) 2006/11/12 05:06:43 vq 1.10.2.1: #i71422# Add a new feature: Using @@ as a recipe prefix redirects the output (stdout and stderr) of a recipe to /dev/null (or NUL on W32) completely suppressing the output of that recipe to the terminal. As for the @ prefix this can be disabled using the -v[r] switch. --- dmake/unix/runargv.c | 466 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 344 insertions(+), 122 deletions(-) (limited to 'dmake/unix') diff --git a/dmake/unix/runargv.c b/dmake/unix/runargv.c index 4d1f56beb6c6..3714e8b107a2 100644 --- a/dmake/unix/runargv.c +++ b/dmake/unix/runargv.c @@ -1,6 +1,6 @@ /* $RCSfile: runargv.c,v $ --- $Revision: 1.10 $ --- last change: $Author: vg $ $Date: 2006-09-25 09:47:57 $ +-- $Revision: 1.11 $ +-- last change: $Author: vg $ $Date: 2007-01-18 09:44:15 $ -- -- SYNOPSIS -- Invoke a sub process. @@ -34,37 +34,78 @@ Exec_commands() [make.c] builds the recipes associated to the given target. Do_cmnd() [sysintf.c] feeds the given command or command group to runargv(). -runargv() [unix/runargv] The actual child processes are started in this - function, even in non parallel builds (MAXPROCESS==1) child processes are - created. - If recipes for a target are currently running attach them to the process - array entry (_procs[i]) of that target and return. - If the maximum number of concurrently running processes is reached - Wait_for_child(?, -1) is used to wait for a process array entry to become +The following flowchart decripes the process flow starting with runargv, +descriptions for each of the functions are following. + + +--------------------------------+ + | runargv | <+ + +--------------------------------+ | + | ^ | + | | returns if | + | calls | wfc is false | + v | | + +--------------------------------+ | + | _add_child | | + +--------------------------------+ | + | ^ | + | calls if | | if another process + | wfc is true | returns | is queued: + v | | recursive call + +--------------------------------+ | + | Wait_for_Child | | + +--------------------------------+ | + | ^ | + | | process queue | + | calls | is empty | + v | | + +--------------------------------+ | + | _finished_child | -+ + +--------------------------------+ + + + +runargv() [unix/runargv] The runargv function manages up to MAXPROCESS + process queues (_procs[i]) for parallel process execution and hands + the actual commands down to the operating system. + Each of the process queues handles the sequential execution of commands + that belong to that process queue. Usually this means the sequential + execution of the recipe lines that belong to one target. + Even in non parallel builds (MAXPROCESS==1) child processes are + created and handled. + If recipes for a target are currently running attach them to the + corresponding process queue (_procs[i]) of that target and return. + If the maximum number (MAXPROCESS) of concurrently running queues is + reached use Wait_for_child(?, -1) to wait for a process queue to become available. New child processes are started using: spawn: posix_spawnp (POSIX) or spawnvp (cygwin). - fork/execvp: Create a client process with fork and run the command with - execvp. + fork/execvp: Create a client process with fork and run the command + with execvp. The parent calls _add_child() to track the child. -_add_child() [unix/runargv] creates a new process array entry and enters the child - parameters. - If Wait_for_completion (global variable) is set the function calls - Wait_for_child to wait for the new process to be finished. - -Wait_for_child(abort_flg, pid) [unix/runargv] waits for the child processes - with pid to finish. All finished processes are handled by calling - _finished_child() for each of them. - If pid == -1 wait for the next child process to finish. - If abort_flg is TRUE no further processes will be added to the process - array. - If the global variable Wait_for_completion is set then all finished - processes are handled until the process with the given pid is reached. - -_finished_child(pid, ?) [unix/runargv] handles the finished child. If there are - more commands in this process array entry start the next with runargv() - otherwise . +_add_child(..., wfc) [unix/runargv] creates (or reuses) a process queue + and enters the child's parameters. + If wfc (wait for completion) is TRUE the function calls + Wait_for_child to wait for the whole process queue to be finished. + +Wait_for_child(abort_flg, pqid) [unix/runargv] waits either for the current + process from process queue pqid to finish or if the W_WFC attribute is + set for all entries of that process queue (recursively) to finish. + All finished processes are handled by calling _finished_child() for each + of them. + If pqid == -1 wait for the next process to finish but honor the A_WFC + attribute of that process (queue) and wait for the whole queue if needed. + If abort_flg is TRUE no further processes will be added to any process + queue. + If a pqid is given but a process from another process queue finishes + first that process is handled and A_WFC is also honored. + All finished processes are processed until the process from the given pqid + is reached or gone (might have been handled while finishing another process + queue). + +_finished_child(pid, ?) [unix/runargv] handles the finished child. If there + are more commands in the corresponding process queue start the next with + runargv(). */ #include @@ -97,9 +138,8 @@ _finished_child(pid, ?) [unix/runargv] handles the finished child. If there are typedef struct prp { char *prp_cmd; int prp_group; - int prp_ignore; + t_attr prp_attr; int prp_last; - int prp_shell; struct prp *prp_next; } RCP, *RCPPTR; @@ -109,18 +149,20 @@ typedef struct pr { CELLPTR pr_target; int pr_ignore; int pr_last; + int pr_wfc; RCPPTR pr_recipe; RCPPTR pr_recipe_end; char *pr_dir; } PR; static PR *_procs = NIL(PR); /* Array to hold concurrent processes. */ +static int _procs_size = 0; /* Savegard to find MAXPROCESS changes. */ static int _proc_cnt = 0; /* Number of running processes. */ static int _abort_flg= FALSE; static int _use_i = -1; -static int _add_child ANSI((int, CELLPTR, int, int)); -static void _attach_cmd ANSI((char *, int, int, CELLPTR, int, int)); +static int _add_child ANSI((int, CELLPTR, int, int, int)); +static void _attach_cmd ANSI((char *, int, CELLPTR, t_attr, int)); static void _finished_child ANSI((int, int)); static int _running ANSI((CELLPTR)); @@ -133,7 +175,7 @@ private_strerror (errnum) #ifdef arm32 extern const char * const sys_errlist[]; #else -#if defined(linux) || defined(__FreeBSD__) +#if defined(linux) || defined(__FreeBSD__) || defined(__OpenBSD__) extern const char * const sys_errlist[]; #else extern char *sys_errlist[]; @@ -150,51 +192,59 @@ private_strerror (errnum) #endif /* HAVE_STRERROR */ PUBLIC int -runargv(target, ignore, group, last, shell, cmd)/* -================================================== - Execute the command given by cmd. */ +runargv(target, group, last, cmnd_attr, cmd)/* +============================================== + Execute the command given by cmd. + + Return 0 if the command executed and finished or + 1 if the command started and is running. + */ CELLPTR target; -int ignore; int group; int last; -int shell; +t_attr cmnd_attr; /* Attributes for current cmnd. */ char *cmd; { + int ignore = (cmnd_attr & A_IGNORE)!= 0; /* Ignore errors ('-'). */ + int shell = (cmnd_attr & A_SHELL) != 0; /* Use shell ('+'). */ + int mute = (cmnd_attr & A_MUTE) != 0; /* Mute output ('@@'). */ + int wfc = (cmnd_attr & A_WFC) != 0; /* Wait for completion. */ + int pid; int st_pq = 0; /* Current _exec_shell target process index */ char **argv; - int old_stdout; /* For internal echo and spawn. */ + int old_stdout = -1; /* For shell escapes and */ + int old_stderr = -1; /* @@-recipe silencing. */ int internal = 0; /* Used to indicate internal command. */ - /* Special handling for the shell function macro is required. If the currend - * command is called as part of a shell escape in a recipe make sure that all - * previous recipe lines of this target have finished. */ + /* Special handling for the shell function macro is required. If the + * currend command is called as part of a shell escape in a recipe make + * sure that all previous recipe lines of this target have finished. */ if( Is_exec_shell ) { if( (st_pq = _running(Shell_exec_target)) != -1 ) { - Wait_for_child(FALSE, _procs[st_pq].pr_pid); + RCPPTR rp; + /* Add WFC to _procs[st_pq]. */ + _procs[st_pq].pr_wfc = TRUE; + /* Set also the A_WFC flag in the recipe attributes. */ + for( rp = _procs[st_pq].pr_recipe ; rp != NIL(RCP); rp = rp->prp_next ) + rp->prp_attr |= A_WFC; + + Wait_for_child(FALSE, st_pq); } } else { if( _running(target) != -1 /*&& Max_proc != 1*/ ) { /* The command will be executed when the previous recipe * line completes. */ - _attach_cmd( cmd, group, ignore, target, last, shell ); + _attach_cmd( cmd, group, target, cmnd_attr, last ); return(1); } } - /* Any Fatal call can potentially loop by recursion because we - * are called from the Quit routine that Fatal indirectly calls - * since Fatal should not happen I have left this bug in here */ - while( _proc_cnt == Max_proc ) { /* This forces sequential execution for Max_proc == 1. */ - if( Wait_for_child(FALSE, -1) == -1 ) { - if( ! in_quit() || errno != ECHILD ) - Fatal( "Lost a child %d: %s", errno, strerror( errno ) ); - else {/* we are quitting and the _proc_cnt was stuffed up by ^C */ - fprintf(stderr,"_proc_cnt %d, Max_proc %d\n",_proc_cnt,Max_proc); - _proc_cnt = 0; - } - } + /* If all process array entries are used wait until we get a free + * slot. For Max_proc == 1 this forces sequential execution. */ + while( _proc_cnt == Max_proc ) { + Wait_for_child(FALSE, -1); } /* remove leading whitespace */ @@ -219,23 +269,37 @@ char *cmd; while( iswhite(*cmd) ) ++cmd; } + /* redirect output for _exec_shell / @@-recipes. */ if( Is_exec_shell ) { + /* Add error checking? */ old_stdout = dup(1); - close(1); - dup( fileno(stdout_redir) ); + dup2( fileno(stdout_redir), 1 ); + } + if( mute ) { + old_stderr = dup(2); + dup2( zerofd, 2 ); + + if( !Is_exec_shell ) { + old_stdout = dup(1); + dup2( zerofd, 1 ); + } } + printf("%s%s", cmd, nl ? "\n" : ""); fflush(stdout); - if( Is_exec_shell ) { - close(1); - dup(old_stdout); + + /* Restore stdout/stderr if needed. */ + if( old_stdout != -1 ) { + dup2(old_stdout, 1); + if( old_stderr != -1 ) + dup2(old_stderr, 2); } internal = 1; } if ( internal ) { /* Use _add_child() / _finished_child() with internal command. */ - int cur_proc = _add_child(-1, target, ignore, last); + int cur_proc = _add_child(-1, target, ignore, last, FALSE); _finished_child(-1, cur_proc); return 0; } @@ -248,9 +312,18 @@ char *cmd; /* As no other childs are started while the output is redirected this * is save. */ if( Is_exec_shell ) { + /* Add error checking? */ old_stdout = dup(1); - close(1); - dup( fileno(stdout_redir) ); + dup2( fileno(stdout_redir), 1 ); + } + if( mute ) { + old_stderr = dup(2); + dup2( zerofd, 2 ); + + if( !Is_exec_shell ) { + old_stdout = dup(1); + dup2( zerofd, 1 ); + } } #if __CYGWIN__ pid = spawnvp(_P_NOWAIT, argv[0], (const char**) argv); @@ -258,17 +331,26 @@ char *cmd; if (posix_spawnp (&pid, argv[0], NULL, NULL, argv, (char *)NULL)) pid = -1; /* posix_spawn failed */ #endif /* __CYGWIN__ */ - if( Is_exec_shell ) { - close(1); - dup(old_stdout); + if( old_stdout != -1 ) { + dup2(old_stdout, 1); + if( old_stderr != -1 ) + dup2(old_stderr, 2); } - if(pid == -1) - { /* spawn failed */ - Error("%s: %s", argv[0], strerror(errno)); - Handle_result(-1, ignore, _abort_flg, target); - return(-1); + if(pid == -1) { + /* spawn failed */ + int continue_status = Continue; + Continue = TRUE; /* survive error message */ + Error("%s: %s", argv[0], strerror(errno)); + Continue = continue_status; + + Handle_result(-1, ignore, _abort_flg, target); + /* Handle_result() aborts dmake if we are not told to + * ignore errors. If we reach the this point return 0 as + * errors are obviously ignored and indicate that the process + * finished. */ + return 0; } else { - _add_child(pid, target, ignore, last); + _add_child(pid, target, ignore, last, wfc); } #else /* ENABLE_SPAWN && ... */ @@ -276,26 +358,40 @@ char *cmd; switch( pid=fork() ){ case -1: /* fork failed */ - Error("%s: %s", argv[0], strerror( errno )); - Handle_result(-1, ignore, _abort_flg, target); - return(-1); + Fatal("fork failed: %s: %s", argv[0], strerror( errno )); case 0: /* child */ - /* redirect stdout for _exec_shell */ + /* redirect output for _exec_shell / @@-recipes. */ if( Is_exec_shell ) { - /* old_stdout = dup(1); */ - close(1); - dup( fileno(stdout_redir) ); + /* Add error checking? */ + old_stdout = dup(1); + dup2( fileno(stdout_redir), 1 ); + } + if( mute ) { + old_stderr = dup(2); + dup2( zerofd, 2 ); + + if( !Is_exec_shell ) { + old_stdout = dup(1); + dup2( zerofd, 1 ); + } } execvp(argv[0], argv); - /* restoring stdout is not needed */ + /* restoring output to catch potential error output if execvp() + * failed. */ + if( old_stdout != -1 ) { + dup2(old_stdout, 1); + if( old_stderr != -1 ) + dup2(old_stderr, 2); + } Continue = TRUE; /* survive error message */ Error("%s: %s", argv[0], strerror( errno )); kill(getpid(), SIGTERM); /*NOTREACHED*/ + Fatal("\nInternal Error - kill could't kill child %d.\n", getpid()); default: /* parent */ - _add_child(pid, target, ignore, last); + _add_child(pid, target, ignore, last, wfc); } #endif /* ENABLE_SPAWN && ... */ @@ -305,24 +401,59 @@ char *cmd; PUBLIC int -Wait_for_child( abort_flg, pid )/* -================================== - Wait for the child processes with pid to to finish. All finished processes - are handled by calling _finished_child() for each of them. - If pid == -1 wait for the next child process to finish. - If abort_flg is TRUE no further processes will be added to the process - array. - If the global variable Wait_for_completion is set then all finished - processes are handled until the process with the given pid is reached. +Wait_for_child( abort_flg, pqid )/* +=================================== + Wait for the next processes from process queue pqid to finish. All finished + processes are handled by calling _finished_child() for each of them. + If pqid == -1 wait for the next process to finish. + If abort_flg is TRUE no further processes will be added to any process + queue. The A_WFC attribute is honored, see the documentation at the top + of this file. + Return 0 if we successfully waited for a process and -1 if there was nothing + to wait for. */ int abort_flg; -int pid; +int pqid; { + int pid; int wid; int status; + /* Never wait for internal commands. */ int waitchild; int is_exec_shell_status = Is_exec_shell; + if( !_procs ) { + /* No process was ever created, i.e. _procs is not yet initialized. + * Nothing to wait for. */ + return -1; + } + + if( pqid > Max_proc ) Fatal("Internal Error: pqid > Max_proc !"); + + if( pqid == -1 ) { + /* Check if there is something to wait for. */ + int i; + for( i=0; i runargv(). */ + if( pid != wid ) { + if( !_procs[pqid].pr_valid || _procs[pqid].pr_pid != pid ) { + /* Someone finished pid, no need to wait further. */ + waitchild = FALSE; + } + } + else + /* We finished pid, no need to wait further. */ + waitchild = FALSE; + } } - while( waitchild && pid != wid ); + while( waitchild ); Is_exec_shell = is_exec_shell_status; return(0); @@ -355,35 +545,50 @@ PUBLIC void Clean_up_processes() { register int i; + int ret; if( _procs != NIL(PR) ) { for( i=0; ipr_target = target; pp->pr_ignore = ignore; pp->pr_last = last; + pp->pr_wfc = wfc; /* Freed above and after the last recipe in _finished child(). */ pp->pr_dir = DmStrDup(Get_current_dir()); @@ -417,10 +623,10 @@ int last; _proc_cnt++; if( pid != -1 ) { - /* Wait for each recipe to finish if Wait_for_completion is TRUE. This + /* Wait for each recipe to finish if wfc is TRUE. This * basically forces sequential execution. */ - if( Wait_for_completion ) - Wait_for_child( FALSE, pid ); + if( wfc ) + Wait_for_child( FALSE, i ); return -1; } else @@ -452,10 +658,16 @@ int status; /* Some children we didn't make esp true if using /bin/sh to execute a * a pipe and feed the output as a makefile into dmake. */ - if( i == Max_proc ) return; + if( i == Max_proc ) { + Warning("Internal Warning: finished pid %d is not in pq!?", pid); + return; + } } - _procs[i].pr_valid = 0; /* Not a running process anymore. */ + /* Not a running process anymore, the next runargv() will not use + * _attach_cmd(). */ + _procs[i].pr_valid = 0; + if( Measure & M_RECIPE ) Do_profile_output( "e", M_RECIPE, _procs[i].pr_target ); @@ -480,17 +692,24 @@ int status; _procs[i].pr_recipe = rp->prp_next; _use_i = i; - /* Run next recipe line. */ - runargv( _procs[i].pr_target, rp->prp_ignore, rp->prp_group, - rp->prp_last, rp->prp_shell, rp->prp_cmd ); + /* Run next recipe line. The rp->prp_attr propagates a possible + * wfc condition. */ + runargv( _procs[i].pr_target, rp->prp_group, + rp->prp_last, rp->prp_attr, rp->prp_cmd ); _use_i = -1; FREE( rp->prp_cmd ); FREE( rp ); + /* If all process queues are used wait for the next process to + * finish. Is this really needed here? */ if( _proc_cnt == Max_proc ) Wait_for_child( FALSE, -1 ); } else { + /* empty the queue on abort. */ + if( _abort_flg ) + _procs[i].pr_recipe = NIL(RCP); + Unlink_temp_files( _procs[i].pr_target ); Handle_result(status,_procs[i].pr_ignore,_abort_flg,_procs[i].pr_target); @@ -529,13 +748,14 @@ CELLPTR cp; static void -_attach_cmd( cmd, group, ignore, cp, last, shell ) +_attach_cmd( cmd, group, cp, cmnd_attr, last )/* +================================================ + Attach to an active process queue. Inherit wfc setting. */ char *cmd; int group; -int ignore; CELLPTR cp; +t_attr cmnd_attr; int last; -int shell; { register int i; RCPPTR rp; @@ -547,10 +767,12 @@ int shell; TALLOC( rp, 1, RCP ); rp->prp_cmd = DmStrDup(cmd); + rp->prp_attr = cmnd_attr; + /* Inherit wfc from process queue. */ + if( _procs[i].pr_wfc ) + rp->prp_attr |= A_WFC; rp->prp_group = group; - rp->prp_ignore= ignore; rp->prp_last = last; - rp->prp_shell = shell; if( _procs[i].pr_recipe == NIL(RCP) ) _procs[i].pr_recipe = _procs[i].pr_recipe_end = rp; -- cgit v1.2.3