001        SUBROUTINE PDDBTRSV( UPLO , TRANS , N , BWL , BWU , NRHS , A , JA , DESCA ,
002       $B , IB , DESCB , AF , LAF , WORK , LWORK , INFO )
003  
004  *     -- ScaLAPACK routine(version 1.7) --
005  *     University of Tennessee , Knoxville , Oak Ridge National Laboratory ,
006  *     and University of California , Berkeley.
007  *     April 3 , 2000
008  
009  *     .. Scalar Arguments ..
010        CHARACTER TRANS , UPLO
011        INTEGER BWL , BWU , IB , INFO , JA , LAF , LWORK , N , NRHS
012  *     ..
013  *     .. Array Arguments ..
014        INTEGER DESCA( * ) , DESCB( * )
015        DOUBLE PRECISION A( * ) , AF( * ) , B( * ) , WORK( * )
016  *     ..
017  
018  *     Purpose
019  *     === ====
020  
021  *     PDDBTRSV solves a banded triangular system of linear equations
022  
023  *     A(1 : N , JA : JA + N - 1) * X = B(IB : IB + N - 1 , 1 : NRHS)
024  *     or
025  *     A(1 : N , JA : JA + N - 1)^T * X = B(IB : IB + N - 1 , 1 : NRHS)
026  
027  *     where A(1 : N , JA : JA + N - 1) is a banded
028  *     triangular matrix factor produced by the
029  *     Gaussian elimination code PD@(dom_pre)BTRF
030  *     and is stored in A(1 : N , JA : JA + N - 1) and AF.
031  *     The matrix stored in A(1 : N , JA : JA + N - 1) is either
032  *     upper or lower triangular according to UPLO ,
033  *     and the choice of solving A(1 : N , JA : JA + N - 1) or A(1 : N , JA : JA + N - 1)^T
034  *     is dictated by the user by the parameter TRANS.
035  
036  *     Routine PDDBTRF MUST be called first.
037  
038  *     === ==================================================================
039  
040  *     Arguments
041  *     === ======
042  
043  *     UPLO(global input) CHARACTER
044  *     = 'U' : Upper triangle of A(1 : N , JA : JA + N - 1) is stored ;
045  *     = 'L' : Lower triangle of A(1 : N , JA : JA + N - 1) is stored.
046  
047  *     TRANS(global input) CHARACTER
048  *     = 'N' : Solve with A(1 : N , JA : JA + N - 1) ;
049  *     = 'T' or 'C' : Solve with A(1 : N , JA : JA + N - 1)^T ;
050  
051  *     N(global input) INTEGER
052  *     The number of rows and columns to be operated on , i.e. the
053  *     order of the distributed submatrix A(1 : N , JA : JA + N - 1). N >= 0.
054  
055  *     BWL(global input) INTEGER
056  *     Number of subdiagonals. 0 <= BWL <= N - 1
057  
058  *     BWU(global input) INTEGER
059  *     Number of superdiagonals. 0 <= BWU <= N - 1
060  
061  *     NRHS(global input) INTEGER
062  *     The number of right hand sides , i.e. , the number of columns
063  *     of the distributed submatrix B(IB : IB + N - 1 , 1 : NRHS).
064  *     NRHS >= 0.
065  
066  *     A(local input / local output) DOUBLE PRECISION pointer into
067  *     local memory to an array with first dimension
068  *     LLD_A >=(bwl + bwu + 1)(stored in DESCA).
069  *     On entry , this array contains the local pieces of the
070  *     N - by - N unsymmetric banded distributed Cholesky factor L or
071  *     L^T A(1 : N , JA : JA + N - 1).
072  *     This local portion is stored in the packed banded format
073  *     used in LAPACK. Please see the Notes below and the
074  *     ScaLAPACK manual for more detail on the format of
075  *     distributed matrices.
076  
077  *     JA(global input) INTEGER
078  *     The index in the global array A that points to the start of
079  *     the matrix to be operated on(which may be either all of A
080  *     or a submatrix of A).
081  
082  *     DESCA(global and local input) INTEGER array of dimension DLEN.
083  *     if 1D type(DTYPE_A = 501) , DLEN >= 7 ;
084  *     if 2D type(DTYPE_A = 1) , DLEN >= 9 .
085  *     The array descriptor for the distributed matrix A.
086  *     Contains information of mapping of A to memory. Please
087  *     see NOTES below for full description and options.
088  
089  *     B(local input / local output) DOUBLE PRECISION pointer into
090  *     local memory to an array of local lead dimension lld_b >= NB.
091  *     On entry , this array contains the
092  *     the local pieces of the right hand sides
093  *     B(IB : IB + N - 1 , 1 : NRHS).
094  *     On exit , this contains the local piece of the solutions
095  *     distributed matrix X.
096  
097  *     IB(global input) INTEGER
098  *     The row index in the global array B that points to the first
099  *     row of the matrix to be operated on(which may be either
100  *     all of B or a submatrix of B).
101  
102  *     DESCB(global and local input) INTEGER array of dimension DLEN.
103  *     if 1D type(DTYPE_B = 502) , DLEN >= 7 ;
104  *     if 2D type(DTYPE_B = 1) , DLEN >= 9.
105  *     The array descriptor for the distributed matrix B.
106  *     Contains information of mapping of B to memory. Please
107  *     see NOTES below for full description and options.
108  
109  *     AF(local output) DOUBLE PRECISION array , dimension LAF.
110  *     Auxiliary Fillin Space.
111  *     Fillin is created during the factorization routine
112  *     PDDBTRF and this is stored in AF. If a linear system
113  *     is to be solved using PDDBTRS after the factorization
114  *     routine , AF *must not be altered* after the factorization.
115  
116  *     LAF(local input) INTEGER
117  *     Size of user - input Auxiliary Fillin space AF. Must be >=
118  *     NB*(bwl + bwu) + 6*max(bwl , bwu)*max(bwl , bwu)
119  *     If LAF is not large enough , an error code will be returned
120  *     and the minimum acceptable size will be returned in AF( 1 )
121  
122  *     WORK(local workspace / local output)
123  *     DOUBLE PRECISION temporary workspace. This space may
124  *     be overwritten in between calls to routines. WORK must be
125  *     the size given in LWORK.
126  *     On exit , WORK( 1 ) contains the minimal LWORK.
127  
128  *     LWORK(local input or global input) INTEGER
129  *     Size of user - input workspace WORK.
130  *     If LWORK is too small , the minimal acceptable size will be
131  *     returned in WORK(1) and an error code is returned. LWORK >=
132  *     (max(bwl , bwu)*NRHS)
133  
134  *     INFO(global output) INTEGER
135  *     = 0 : successful exit
136  *     < 0 : If the i - th argument is an array and the j - entry had
137  *     an illegal value , then INFO = - (i*100 + j) , if the i - th
138  *     argument is a scalar and had an illegal value , then
139  *     INFO = - i.
140  
141  *     === ==================================================================
142  
143  *     Restrictions
144  *     === =========
145  
146  *     The following are restrictions on the input parameters. Some of these
147  *     are temporary and will be removed in future releases , while others
148  *     may reflect fundamental technical limitations.
149  
150  *     Non - cyclic restriction : VERY IMPORTANT !
151  *     P*NB >= mod(JA - 1 , NB) + N.
152  *     The mapping for matrices must be blocked , reflecting the nature
153  *     of the divide and conquer algorithm as a task - parallel algorithm.
154  *     This formula in words is : no processor may have more than one
155  *     chunk of the matrix.
156  
157  *     Blocksize cannot be too small :
158  *     If the matrix spans more than one processor , the following
159  *     restriction on NB , the size of each block on each processor ,
160  *     must hold :
161  *     NB >= 2*MAX(BWL , BWU)
162  *     The bulk of parallel computation is done on the matrix of size
163  *     O(NB) on each processor. If this is too small , divide and conquer
164  *     is a poor choice of algorithm.
165  
166  *     Submatrix reference :
167  *     JA = IB
168  *     Alignment restriction that prevents unnecessary communication.
169  
170  *     === ==================================================================
171  
172  *     Notes
173  *     === ==
174  
175  *     If the factorization routine and the solve routine are to be called
176  *     separately(to solve various sets of righthand sides using the same
177  *     coefficient matrix) , the auxiliary space AF *must not be altered*
178  *     between calls to the factorization routine and the solve routine.
179  
180  *     The best algorithm for solving banded and tridiagonal linear systems
181  *     depends on a variety of parameters , especially the bandwidth.
182  *     Currently , only algorithms designed for the case N / P >> bw are
183  *     implemented. These go by many names , including Divide and Conquer ,
184  *     Partitioning , domain decomposition - type , etc.
185  
186  *     Algorithm description : Divide and Conquer
187  
188  *     The Divide and Conqer algorithm assumes the matrix is narrowly
189  *     banded compared with the number of equations. In this situation ,
190  *     it is best to distribute the input matrix A one - dimensionally ,
191  *     with columns atomic and rows divided amongst the processes.
192  *     The basic algorithm divides the banded matrix up into
193  *     P pieces with one stored on each processor ,
194  *     and then proceeds in 2 phases for the factorization or 3 for the
195  *     solution of a linear system.
196  *     1) Local Phase :
197  *     The individual pieces are factored independently and in
198  *     parallel. These factors are applied to the matrix creating
199  *     fillin , which is stored in a non - inspectable way in auxiliary
200  *     space AF. Mathematically , this is equivalent to reordering
201  *     the matrix A as P A P^T and then factoring the principal
202  *     leading submatrix of size equal to the sum of the sizes of
203  *     the matrices factored on each processor. The factors of
204  *     these submatrices overwrite the corresponding parts of A
205  *     in memory.
206  *     2) Reduced System Phase :
207  *     A small(max(bwl , bwu)* (P - 1)) system is formed representing
208  *     interaction of the larger blocks , and is stored(as are its
209  *     factors) in the space AF. A parallel Block Cyclic Reduction
210  *     algorithm is used. For a linear system , a parallel front solve
211  *     followed by an analagous backsolve , both using the structure
212  *     of the factored matrix , are performed.
213  *     3) Backsubsitution Phase :
214  *     For a linear system , a local backsubstitution is performed on
215  *     each processor in parallel.
216  
217  *     Descriptors
218  *     === ========
219  
220  *     Descriptors now have *types* and differ from ScaLAPACK 1.0.
221  
222  *     Note : banded codes can use either the old two dimensional
223  *     or new one - dimensional descriptors , though the processor grid in
224  *     both cases *must be one - dimensional*. We describe both types below.
225  
226  *     Each global data object is described by an associated description
227  *     vector. This vector stores the information required to establish
228  *     the mapping between an object element and its corresponding process
229  *     and memory location.
230  
231  *     Let A be a generic term for any 2D block cyclicly distributed array.
232  *     Such a global array has an associated description vector DESCA.
233  *     In the following comments , the character _ should be read as
234  *     "of the global array".
235  
236  *     NOTATION STORED IN EXPLANATION
237  *     --- ------------ -------------- --------------------------------------
238  *     DTYPE_A(global) DESCA( DTYPE_ )The descriptor type. In this case ,
239  *     DTYPE_A = 1.
240  *     CTXT_A(global) DESCA( CTXT_ ) The BLACS context handle , indicating
241  *     the BLACS process grid A is distribu -
242  *     ted over. The context itself is glo -
243  *     bal , but the handle(the integer
244  *     value) may vary.
245  *     M_A(global) DESCA( M_ ) The number of rows in the global
246  *     array A.
247  *     N_A(global) DESCA( N_ ) The number of columns in the global
248  *     array A.
249  *     MB_A(global) DESCA( MB_ ) The blocking factor used to distribute
250  *     the rows of the array.
251  *     NB_A(global) DESCA( NB_ ) The blocking factor used to distribute
252  *     the columns of the array.
253  *     RSRC_A(global) DESCA( RSRC_ ) The process row over which the first
254  *     row of the array A is distributed.
255  *     CSRC_A(global) DESCA( CSRC_ ) The process column over which the
256  *     first column of the array A is
257  *     distributed.
258  *     LLD_A(local) DESCA( LLD_ ) The leading dimension of the local
259  *     array. LLD_A >= MAX(1 , LOCr(M_A)).
260  
261  *     Let K be the number of rows or columns of a distributed matrix ,
262  *     and assume that its process grid has dimension p x q.
263  *     LOCr( K ) denotes the number of elements of K that a process
264  *     would receive if K were distributed over the p processes of its
265  *     process column.
266  *     Similarly , LOCc( K ) denotes the number of elements of K that a
267  *     process would receive if K were distributed over the q processes of
268  *     its process row.
269  *     The values of LOCr() and LOCc() may be determined via a call to the
270  *     ScaLAPACK tool function , NUMROC :
271  *     LOCr( M ) = NUMROC( M , MB_A , MYROW , RSRC_A , NPROW ) ,
272  *     LOCc( N ) = NUMROC( N , NB_A , MYCOL , CSRC_A , NPCOL ).
273  *     An upper bound for these quantities may be computed by :
274  *     LOCr( M ) <= ceil( ceil(M / MB_A) / NPROW )*MB_A
275  *     LOCc( N ) <= ceil( ceil(N / NB_A) / NPCOL )*NB_A
276  
277  *     One - dimensional descriptors :
278  
279  *     One - dimensional descriptors are a new addition to ScaLAPACK since
280  *     version 1.0. They simplify and shorten the descriptor for 1D
281  *     arrays.
282  
283  *     Since ScaLAPACK supports two - dimensional arrays as the fundamental
284  *     object , we allow 1D arrays to be distributed either over the
285  *     first dimension of the array(as if the grid were P - by - 1) or the
286  *     2nd dimension(as if the grid were 1 - by - P). This choice is
287  *     indicated by the descriptor type(501 or 502)
288  *     as described below.
289  
290  *     IMPORTANT NOTE : the actual BLACS grid represented by the
291  *     CTXT entry in the descriptor may be *either* P - by - 1 or 1 - by - P
292  *     irrespective of which one - dimensional descriptor type
293  *     (501 or 502) is input.
294  *     This routine will interpret the grid properly either way.
295  *     ScaLAPACK routines *do not support intercontext operations* so that
296  *     the grid passed to a single ScaLAPACK routine *must be the same*
297  *     for all array descriptors passed to that routine.
298  
299  *     NOTE : In all cases where 1D descriptors are used , 2D descriptors
300  *     may also be used , since a one - dimensional array is a special case
301  *     of a two - dimensional array with one dimension of size unity.
302  *     The two - dimensional array used in this case *must* be of the
303  *     proper orientation :
304  *     If the appropriate one - dimensional descriptor is DTYPEA = 501
305  *     (1 by P type) , then the two dimensional descriptor must
306  *     have a CTXT value that refers to a 1 by P BLACS grid ;
307  *     If the appropriate one - dimensional descriptor is DTYPEA = 502
308  *     (P by 1 type) , then the two dimensional descriptor must
309  *     have a CTXT value that refers to a P by 1 BLACS grid.
310  
311  *     Summary of allowed descriptors , types , and BLACS grids :
312  *     DTYPE 501 502 1 1
313  *     BLACS grid 1xP or Px1 1xP or Px1 1xP Px1
314  *     --- --------------------------------------------------
315  *     A               OK NO OK NO
316  *     B               NO OK NO OK
317  
318  *     Note that a consequence of this chart is that it is not possible
319  *     for *both* DTYPE_A and DTYPE_B to be 2D_type(1) , as these lead
320  *     to opposite requirements for the orientation of the BLACS grid ,
321  *     and as noted before , the *same* BLACS context must be used in
322  *     all descriptors in a single ScaLAPACK subroutine call.
323  
324  *     Let A be a generic term for any 1D block cyclicly distributed array.
325  *     Such a global array has an associated description vector DESCA.
326  *     In the following comments , the character _ should be read as
327  *     "of the global array".
328  
329  *     NOTATION STORED IN EXPLANATION
330  *     --- ------------ ---------- ------------------------------------------
331  *     DTYPE_A(global) DESCA( 1 ) The descriptor type. For 1D grids ,
332  *     TYPE_A = 501 : 1 - by - P grid.
333  *     TYPE_A = 502 : P - by - 1 grid.
334  *     CTXT_A(global) DESCA( 2 ) The BLACS context handle , indicating
335  *     the BLACS process grid A is distribu -
336  *     ted over. The context itself is glo -
337  *     bal , but the handle(the integer
338  *     value) may vary.
339  *     N_A(global) DESCA( 3 ) The size of the array dimension being
340  *     distributed.
341  *     NB_A(global) DESCA( 4 ) The blocking factor used to distribute
342  *     the distributed dimension of the array.
343  *     SRC_A(global) DESCA( 5 ) The process row or column over which the
344  *     first row or column of the array
345  *     is distributed.
346  *     LLD_A(local) DESCA( 6 ) The leading dimension of the local array
347  *     storing the local blocks of the distri -
348  *     buted array A. Minimum value of LLD_A
349  *     depends on TYPE_A.
350  *     TYPE_A = 501 : LLD_A >=
351  *     size of undistributed dimension , 1.
352  *     TYPE_A = 502 : LLD_A >= NB_A , 1.
353  *     Reserved DESCA( 7 ) Reserved for future use.
354  
355  *     === ==================================================================
356  
357  *     Code Developer : Andrew J. Cleary , University of Tennessee.
358  *     Current address : Lawrence Livermore National Labs.
359  *     Last modified by : Peter Arbenz , Institute of Scientific Computing ,
360  *     ETH , Zurich.
361  
362  *     === ==================================================================
363  
364  *     .. Parameters ..
365        DOUBLE PRECISION ONE
366        PARAMETER( ONE = 1.0D + 0 )
367        DOUBLE PRECISION ZERO
368        PARAMETER( ZERO = 0.0D + 0 )
369        INTEGER INT_ONE
370        PARAMETER( INT_ONE = 1 )
371        INTEGER DESCMULT , BIGNUM
372        PARAMETER( DESCMULT = 100 , BIGNUM = DESCMULT*DESCMULT )
373        INTEGER BLOCK_CYCLIC_2D , CSRC_ , CTXT_ , DLEN_ , DTYPE_ ,
374       $LLD_ , MB_ , M_ , NB_ , N_ , RSRC_
375        PARAMETER( BLOCK_CYCLIC_2D = 1 , DLEN_ = 9 , DTYPE_ = 1 ,
376       $CTXT_ = 2 , M_ = 3 , N_ = 4 , MB_ = 5 , NB_ = 6 ,
377       $RSRC_ = 7 , CSRC_ = 8 , LLD_ = 9 )
378        CALL DLACPY( 'N' , BWL , NRHS , B( PART_OFFSET + ODD_SIZE + 1 ) ,
379       $LLDB , WORK( 1 + MAX_BW - BWL ) , MAX_BW )
380  
381        CALL DTRMM( 'L' , 'U' , 'T' , 'N' , BWL , NRHS , - ONE ,
382       $A(( OFST + ( BWL + BWU + 1 ) + ( ODD_SIZE - BWL )*
383       $LLDA ) ) , LLDA - 1 , WORK( 1 + MAX_BW - BWL ) ,
384       $MAX_BW )
385  
386        CALL DMATADD( BWL , NRHS , ONE , WORK( 1 + MAX_BW - BWL ) ,
387       $MAX_BW , ONE , B( PART_OFFSET + ODD_SIZE - BWL +
388       $1 ) , LLDB )
389  
390        END IF
391  
392  *     Use main partition in each processor to solve locally
393  
394        CALL DTBTRS( UPLO , 'T' , 'U' , ODD_SIZE , BWL , NRHS ,
395       $A( OFST + 1 + BWU ) , LLDA , B( PART_OFFSET + 1 ) ,
396       $LLDB , INFO )
397  
398        END IF
399  *     End of "IF( LSAME( TRANS, 'N' ) )"...
400  
401        ELSE
402  *         **************************************************************
403  *         CASE UPLO = 'U' *
404  *         **************************************************************
404               
405            IF( LSAME( TRANS , 'T' ) ) THEN
406  
407  *             Frontsolve
408  
409  *             *****************************************
410  *             Local computation phase
411  *             *****************************************
412  
413  *             Use main partition in each processor to solve locally
414  
414                   
415                CALL DTBTRS( UPLO , 'T' , 'N' , ODD_SIZE , BWU , NRHS ,
416       $        A( OFST + 1 ) , LLDA , B( PART_OFFSET + 1 ) , LLDB ,
417       $        INFO )
418  
419                IF( MYCOL.LT.NP - 1 ) THEN
420  *                 Use factorization of odd - even connection block to modify
421  *                 locally stored portion of right hand side(s)
422  
423  *                 First copy and multiply it into temporary storage ,
424  *                 then use it on RHS
425  
425                       
426                    CALL DLACPY( 'N' , BWU , NRHS ,
427       $            B( PART_OFFSET + ODD_SIZE - BWU + 1 ) , LLDB ,
428       $            WORK( 1 ) , MAX_BW )
429  
430                    CALL DTRMM( 'L' , 'L' , 'T' , 'N' , BWU , NRHS , - ONE ,
431       $            A(( OFST + 1 + ODD_SIZE*LLDA ) ) , LLDA - 1 ,
432       $            WORK( 1 ) , MAX_BW )
433  
434                    CALL DMATADD( BWU , NRHS , ONE , WORK( 1 ) , MAX_BW , ONE ,
435       $            B( PART_OFFSET + ODD_SIZE + 1 ) , LLDB )
436  
437                END IF
438  
439  *             Clear garbage out of workspace block
440  
441                DO 100 IDUM1 = 1 , WORK_SIZE_MIN
441                       
442                    WORK( IDUM1 ) = 0.0
443    100         CONTINUE
444  
444                   
445                IF( MYCOL.NE.0 ) THEN
446  *                 Use the "spike" fillin to calculate contribution to previous
447  *                 processor's righthand - side.
448  
448                       
449                    CALL DGEMM( 'N' , 'N' , BWL , NRHS , ODD_SIZE , - ONE ,
450       $            AF( WORK_U + 1 ) , BWL , B( PART_OFFSET + 1 ) ,
451       $            LLDB , ZERO , WORK( 1 + MAX_BW - BWL ) , MAX_BW )
452                END IF
453  
454  *             ***********************************************
455  *             Formation and solution of reduced system
456  *             ***********************************************
457  
458  *             Send modifications to prior processor's right hand sides
459  
460                IF( MYCOL.GT.0 ) THEN
461  
461                       
462                    CALL DGESD2D( ICTXT , MAX_BW , NRHS , WORK( 1 ) , MAX_BW , 0 ,
463       $            MYCOL - 1 )
464  
465                END IF
466  
467  *             Receive modifications to processor's right hand sides
468  
469                IF( MYCOL.LT.NPCOL - 1 ) THEN
470  
470                       
471                    CALL DGERV2D( ICTXT , MAX_BW , NRHS , WORK( 1 ) , MAX_BW , 0 ,
472       $            MYCOL + 1 )
473  
474  *                 Combine contribution to locally stored right hand sides
475  
476                    CALL DMATADD( MAX_BW , NRHS , ONE , WORK( 1 ) , MAX_BW , ONE ,
477       $            B( PART_OFFSET + ODD_SIZE + 1 ) , LLDB )
478  
479                END IF
480  
481  *             The last processor does not participate in the solution of the
482  *             reduced system , having sent its contribution already.
483                IF( MYCOL.EQ.NPCOL - 1 ) THEN
483                       
484                    GO TO 130
485                END IF
486  
487  *             *************************************
488  *             Modification Loop
489  
490  *             The distance for sending and receiving for each level starts
491  *             at 1 for the first level.
492                LEVEL_DIST = 1
493  
494  *             Do until this proc is needed to modify other procs' equations
495  
496    110 CONTINUE
497        IF( MOD(( MYCOL + 1 ) / LEVEL_DIST , 2 ).NE.0 )
497               
498       $    GO TO 120
499  
500  *         Receive and add contribution to righthand sides from left
501  
502            IF( MYCOL - LEVEL_DIST.GE.0 ) THEN
503  
503                   
504                CALL DGERV2D( ICTXT , MAX_BW , NRHS , WORK( 1 ) , MAX_BW , 0 ,
505       $        MYCOL - LEVEL_DIST )
506  
507                CALL DMATADD( MAX_BW , NRHS , ONE , WORK( 1 ) , MAX_BW , ONE ,
508       $        B( PART_OFFSET + ODD_SIZE + 1 ) , LLDB )
509  
510            END IF
511  
512  *         Receive and add contribution to righthand sides from right
513  
514            IF( MYCOL + LEVEL_DIST.LT.NPCOL - 1 ) THEN
515  
515                   
516                CALL DGERV2D( ICTXT , MAX_BW , NRHS , WORK( 1 ) , MAX_BW , 0 ,
517       $        MYCOL + LEVEL_DIST )
518  
519                CALL DMATADD( MAX_BW , NRHS , ONE , WORK( 1 ) , MAX_BW , ONE ,
520       $        B( PART_OFFSET + ODD_SIZE + 1 ) , LLDB )
521  
522            END IF
523  
524            LEVEL_DIST = LEVEL_DIST*2
525  
526            GO TO 110
527    120 CONTINUE
528  *     [End of GOTO Loop]
529  
530  *     *********************************
531  *     Calculate and use this proc's blocks to modify other procs
532  
533  *     Solve with diagonal block
534  
535        CALL DTBTRS( 'U' , 'T' , 'N' , MAX_BW , MIN( BWU , MAX_BW - 1 ) ,
536       $NRHS , AF( ODD_SIZE*BWU + MBW2 + 1 - MIN( BWU ,
537       $MAX_BW - 1 ) ) , MAX_BW + 1 ,
538       $B( PART_OFFSET + ODD_SIZE + 1 ) , LLDB , INFO )
539  
540        IF( INFO.NE.0 ) THEN
540               
541            GO TO 190
542        END IF
543  
544  *     *********
545        IF( MYCOL / LEVEL_DIST.LE.( NPCOL - 1 ) / LEVEL_DIST - 2 ) THEN
546  
547  *         Calculate contribution from this block to next diagonal block
548  
548               
549            CALL DGEMM( 'T' , 'N' , MAX_BW , NRHS , MAX_BW , - ONE ,
550       $    AF( WORK_U + ( ODD_SIZE )*BWL + 1 ) , MAX_BW ,
551       $    B( PART_OFFSET + ODD_SIZE + 1 ) , LLDB , ZERO ,
552       $    WORK( 1 ) , MAX_BW )
553  
554  *         Send contribution to diagonal block's owning processor.
555  
556            CALL DGESD2D( ICTXT , MAX_BW , NRHS , WORK( 1 ) , MAX_BW , 0 ,
557       $    MYCOL + LEVEL_DIST )
558  
559        END IF
560  *     End of "if( mycol/level_dist .le.(npcol-1)/level_dist-2 )..."
561  
562  *     ************
563        IF(( MYCOL / LEVEL_DIST.GT.0 ) .AND.
564       $( MYCOL / LEVEL_DIST.LE.( NPCOL - 1 ) / LEVEL_DIST - 1 ) )
565       $THEN
566  
567  *     Use offdiagonal block to calculate modification to diag block
568  *     of processor to the left
569  
570        CALL DGEMM( 'N' , 'N' , MAX_BW , NRHS , MAX_BW , - ONE ,
571       $AF( WORK_U + ODD_SIZE*BWL + 2*MBW2 + 1 ) , MAX_BW ,
572       $B( PART_OFFSET + ODD_SIZE + 1 ) , LLDB , ZERO ,
573       $WORK( 1 ) , MAX_BW )
574  
575  *     Send contribution to diagonal block's owning processor.
576  
577        CALL DGESD2D( ICTXT , MAX_BW , NRHS , WORK( 1 ) , MAX_BW , 0 ,
578       $MYCOL - LEVEL_DIST )
579  
580        END IF
581  *     End of "if( mycol/level_dist.le.(npcol-1)/level_dist -1 )..."
582  
583    130 CONTINUE
584  
585        ELSE
586  
587  *         ******************* BACKSOLVE *************************************
588  
589  *         *******************************************************************
590  *         .. Begin reduced system phase of algorithm ..
591  *         *******************************************************************
592  
593  *         The last processor does not participate in the solution of the
594  *         reduced system and just waits to receive its solution.
594               
595            IF( MYCOL.EQ.NPCOL - 1 ) THEN
595                   
596                GO TO 180
597            END IF
598  
599  *         Determine number of steps in tree loop
600  
601            LEVEL_DIST = 1
602    140 CONTINUE
603        IF( MOD(( MYCOL + 1 ) / LEVEL_DIST , 2 ).NE.0 )
603               
604       $    GO TO 150
605  
606            LEVEL_DIST = LEVEL_DIST*2
607  
608            GO TO 140
609    150 CONTINUE
610  
611        IF(( MYCOL / LEVEL_DIST.GT.0 ) .AND.
612       $( MYCOL / LEVEL_DIST.LE.( NPCOL - 1 ) / LEVEL_DIST - 1 ) )
613       $THEN
614  
615  *     Receive solution from processor to left
616  
617        CALL DGERV2D( ICTXT , MAX_BW , NRHS , WORK( 1 ) , MAX_BW , 0 ,
618       $MYCOL - LEVEL_DIST )
619  
620  *     Use offdiagonal block to calculate modification to RHS stored
621  *     on this processor
622  
623        CALL DGEMM( 'T' , 'N' , MAX_BW , NRHS , MAX_BW , - ONE ,
624       $AF( WORK_U + ODD_SIZE*BWL + 2*MBW2 + 1 ) , MAX_BW ,
625       $WORK( 1 ) , MAX_BW , ONE ,
626       $B( PART_OFFSET + ODD_SIZE + 1 ) , LLDB )
627        END IF
628  *     End of "if( mycol/level_dist.le.(npcol-1)/level_dist -1 )..."
629  
630        IF( MYCOL / LEVEL_DIST.LE.( NPCOL - 1 ) / LEVEL_DIST - 2 ) THEN
631  
632  *         Receive solution from processor to right
633  
633               
634            CALL DGERV2D( ICTXT , MAX_BW , NRHS , WORK( 1 ) , MAX_BW , 0 ,
635       $    MYCOL + LEVEL_DIST )
636  
637  *         Calculate contribution from this block to next diagonal block
638  
639            CALL DGEMM( 'N' , 'N' , MAX_BW , NRHS , MAX_BW , - ONE ,
640       $    AF( WORK_U + ( ODD_SIZE )*BWL + 1 ) , MAX_BW ,
641       $    WORK( 1 ) , MAX_BW , ONE ,
642       $    B( PART_OFFSET + ODD_SIZE + 1 ) , LLDB )
643  
644        END IF
645  *     End of "if( mycol/level_dist .le.(npcol-1)/level_dist-2 )..."
646  
647  *     Solve with diagonal block
648  
649        CALL DTBTRS( 'U' , 'N' , 'N' , MAX_BW , MIN( BWU , MAX_BW - 1 ) ,
650       $NRHS , AF( ODD_SIZE*BWU + MBW2 + 1 - MIN( BWU ,
651       $MAX_BW - 1 ) ) , MAX_BW + 1 ,
652       $B( PART_OFFSET + ODD_SIZE + 1 ) , LLDB , INFO )
653  
654        IF( INFO.NE.0 ) THEN
654               
655            GO TO 190
656        END IF
657  
658  *     **Modification Loop *******
659  
660    160 CONTINUE
661        IF( LEVEL_DIST.EQ.1 )
661               
662       $    GO TO 170
663  
664            LEVEL_DIST = LEVEL_DIST / 2
665  
666  *         Send solution to the right
667  
668            IF( MYCOL + LEVEL_DIST.LT.NPCOL - 1 ) THEN
669  
669                   
670                CALL DGESD2D( ICTXT , MAX_BW , NRHS ,
671       $        B( PART_OFFSET + ODD_SIZE + 1 ) , LLDB , 0 ,
672       $        MYCOL + LEVEL_DIST )
673  
674            END IF
675  
676  *         Send solution to left
677  
678            IF( MYCOL - LEVEL_DIST.GE.0 ) THEN
679  
679                   
680                CALL DGESD2D( ICTXT , MAX_BW , NRHS ,
681       $        B( PART_OFFSET + ODD_SIZE + 1 ) , LLDB , 0 ,
682       $        MYCOL - LEVEL_DIST )
683  
684            END IF
685  
686            GO TO 160
687    170 CONTINUE
688  *     [End of GOTO Loop]
689  
690    180 CONTINUE
691  *     [Processor npcol - 1 jumped to here to await next stage]
692  
693  *     ******************************
694  *     Reduced system has been solved , communicate solutions to nearest
695  *     neighbors in preparation for local computation phase.
696  
697  *     Send elements of solution to next proc
698  
699        IF( MYCOL.LT.NPCOL - 1 ) THEN
700  
700               
701            CALL DGESD2D( ICTXT , MAX_BW , NRHS ,
702       $    B( PART_OFFSET + ODD_SIZE + 1 ) , LLDB , 0 ,
703       $    MYCOL + 1 )
704  
705        END IF
706  
707  *     Receive modifications to processor's right hand sides
708  
709        IF( MYCOL.GT.0 ) THEN
710  
710               
711            CALL DGERV2D( ICTXT , MAX_BW , NRHS , WORK( 1 ) , MAX_BW , 0 ,
712       $    MYCOL - 1 )
713  
714        END IF
715  
716  *     *********************************************
717  *     Local computation phase
718  *     *********************************************
719  
720        IF( MYCOL.NE.0 ) THEN
721  *         Use the "spike" fillin to calculate contribution from previous
722  *         processor's solution.
723  
723               
724            CALL DGEMM( 'T' , 'N' , ODD_SIZE , NRHS , BWL , - ONE ,
725       $    AF( WORK_U + 1 ) , BWL , WORK( 1 + MAX_BW - BWL ) ,
726       $    MAX_BW , ONE , B( PART_OFFSET + 1 ) , LLDB )
727  
728        END IF
729  
730        IF( MYCOL.LT.NP - 1 ) THEN
731  *         Use factorization of odd - even connection block to modify
732  *         locally stored portion of right hand side(s)
733  
734  *         First copy and multiply it into temporary storage ,
735  *         then use it on RHS
736  
736               
737            CALL DLACPY( 'N' , BWU , NRHS , B( PART_OFFSET + ODD_SIZE + 1 ) ,
738       $    LLDB , WORK( 1 + MAX_BW - BWU ) , MAX_BW + BWL )
739  
740            CALL DTRMM( 'L' , 'L' , 'N' , 'N' , BWU , NRHS , - ONE ,
741       $    A(( OFST + 1 + ODD_SIZE*LLDA ) ) , LLDA - 1 ,
742       $    WORK( 1 + MAX_BW - BWU ) , MAX_BW + BWL )
743  
744            CALL DMATADD( BWU , NRHS , ONE , WORK( 1 + MAX_BW - BWU ) ,
745       $    MAX_BW + BWL , ONE , B( PART_OFFSET + ODD_SIZE -
746       $    BWU + 1 ) , LLDB )
747  
748        END IF
749  
750  *     Use main partition in each processor to solve locally
751  
752        CALL DTBTRS( UPLO , 'N' , 'N' , ODD_SIZE , BWU , NRHS ,
753       $A( OFST + 1 ) , LLDA , B( PART_OFFSET + 1 ) , LLDB ,
754       $INFO )
755  
756        END IF
757  *     End of "IF( LSAME( TRANS, 'N' ) )"...
758  
759        END IF
760  *     End of "IF( LSAME( UPLO, 'L' ) )"...
761    190 CONTINUE
762  
763  *     Free BLACS space used to hold standard - form grid.
764  
765        IF( ICTXT_SAVE.NE.ICTXT_NEW ) THEN
765               
766            CALL BLACS_GRIDEXIT( ICTXT_NEW )
767        END IF
768  
769    200 CONTINUE
770  
771  *     Restore saved input parameters
772  
773        ICTXT = ICTXT_SAVE
774        NP = NP_SAVE
775  
776  *     Output minimum worksize
777  
778        WORK( 1 ) = WORK_SIZE_MIN
779  
780        RETURN
781  
782  *     End of PDDBTRSV
783  
784        END386192