Actual source code: bjacobi.c

  1: #define PETSCKSP_DLL

  3: /*
  4:    Defines a block Jacobi preconditioner.
  5: */
 6:  #include src/mat/matimpl.h
 7:  #include private/pcimpl.h
 8:  #include src/ksp/pc/impls/bjacobi/bjacobi.h

 10: static PetscErrorCode PCSetUp_BJacobi_Singleblock(PC,Mat,Mat);
 11: static PetscErrorCode PCSetUp_BJacobi_Multiblock(PC,Mat,Mat);

 15: static PetscErrorCode PCSetUp_BJacobi(PC pc)
 16: {
 17:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;
 18:   Mat            mat = pc->mat,pmat = pc->pmat;
 19:   PetscErrorCode ierr,(*f)(Mat,PetscTruth*,MatReuse,Mat*);
 20:   PetscInt       N,M,start,i,sum,end;
 21:   PetscInt       bs,i_start=-1,i_end=-1;
 22:   PetscMPIInt    rank,size;
 23:   const char     *pprefix,*mprefix;

 26:   MPI_Comm_rank(pc->comm,&rank);
 27:   MPI_Comm_size(pc->comm,&size);
 28:   MatGetLocalSize(pc->pmat,&M,&N);
 29:   MatGetBlockSize(pc->pmat,&bs);

 31:   /* ----------
 32:       Determines the number of blocks assigned to each processor 
 33:   */

 35:   /*   local block count  given */
 36:   if (jac->n_local > 0 && jac->n < 0) {
 37:     MPI_Allreduce(&jac->n_local,&jac->n,1,MPIU_INT,MPI_SUM,pc->comm);
 38:     if (jac->l_lens) { /* check that user set these correctly */
 39:       sum = 0;
 40:       for (i=0; i<jac->n_local; i++) {
 41:         if (jac->l_lens[i]/bs*bs !=jac->l_lens[i]) {
 42:           SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
 43:         }
 44:         sum += jac->l_lens[i];
 45:       }
 46:       if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Local lens sent incorrectly");
 47:     } else {
 48:       PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
 49:       for (i=0; i<jac->n_local; i++) {
 50:         jac->l_lens[i] = bs*((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i));
 51:       }
 52:     }
 53:   } else if (jac->n > 0 && jac->n_local < 0) { /* global block count given */
 54:     /* global blocks given: determine which ones are local */
 55:     if (jac->g_lens) {
 56:       /* check if the g_lens is has valid entries */
 57:       for (i=0; i<jac->n; i++) {
 58:         if (!jac->g_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Zero block not allowed");
 59:         if (jac->g_lens[i]/bs*bs != jac->g_lens[i]) {
 60:           SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
 61:         }
 62:       }
 63:       if (size == 1) {
 64:         jac->n_local = jac->n;
 65:         PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
 66:         PetscMemcpy(jac->l_lens,jac->g_lens,jac->n_local*sizeof(PetscInt));
 67:         /* check that user set these correctly */
 68:         sum = 0;
 69:         for (i=0; i<jac->n_local; i++) sum += jac->l_lens[i];
 70:         if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Global lens sent incorrectly");
 71:       } else {
 72:         MatGetOwnershipRange(pc->pmat,&start,&end);
 73:         /* loop over blocks determing first one owned by me */
 74:         sum = 0;
 75:         for (i=0; i<jac->n+1; i++) {
 76:           if (sum == start) { i_start = i; goto start_1;}
 77:           if (i < jac->n) sum += jac->g_lens[i];
 78:         }
 79:         SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizes\n\
 80:                    used in PCBJacobiSetTotalBlocks()\n\
 81:                    are not compatible with parallel matrix layout");
 82:  start_1:
 83:         for (i=i_start; i<jac->n+1; i++) {
 84:           if (sum == end) { i_end = i; goto end_1; }
 85:           if (i < jac->n) sum += jac->g_lens[i];
 86:         }
 87:         SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizes\n\
 88:                       used in PCBJacobiSetTotalBlocks()\n\
 89:                       are not compatible with parallel matrix layout");
 90:  end_1:
 91:         jac->n_local = i_end - i_start;
 92:         PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
 93:         PetscMemcpy(jac->l_lens,jac->g_lens+i_start,jac->n_local*sizeof(PetscInt));
 94:       }
 95:     } else { /* no global blocks given, determine then using default layout */
 96:       jac->n_local = jac->n/size + ((jac->n % size) > rank);
 97:       PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
 98:       for (i=0; i<jac->n_local; i++) {
 99:         jac->l_lens[i] = ((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i))*bs;
100:         if (!jac->l_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Too many blocks given");
101:       }
102:     }
103:   } else if (jac->n < 0 && jac->n_local < 0) { /* no blocks given */
104:     jac->n         = size;
105:     jac->n_local   = 1;
106:     PetscMalloc(sizeof(PetscInt),&jac->l_lens);
107:     jac->l_lens[0] = M;
108:   }

110:   MPI_Comm_size(pc->comm,&size);
111:   PetscObjectQueryFunction((PetscObject)pc->mat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
112:   if (size == 1 && !f) {
113:     mat  = pc->mat;
114:     pmat = pc->pmat;
115:   } else {
116:     PetscTruth iscopy;
117:     MatReuse   scall;

119:     if (jac->use_true_local) {
120:       scall = MAT_INITIAL_MATRIX;
121:       if (pc->setupcalled) {
122:         if (pc->flag == SAME_NONZERO_PATTERN) {
123:           if (jac->tp_mat) {
124:             scall = MAT_REUSE_MATRIX;
125:             mat   = jac->tp_mat;
126:           }
127:         } else {
128:           if (jac->tp_mat)  {
129:             MatDestroy(jac->tp_mat);
130:           }
131:         }
132:       }
133:       if (!f) {
134:         SETERRQ(PETSC_ERR_SUP,"This matrix does not support getting diagonal block");
135:       }
136:       (*f)(pc->mat,&iscopy,scall,&mat);
137:       /* make submatrix have same prefix as entire matrix */
138:       PetscObjectGetOptionsPrefix((PetscObject)pc->mat,&mprefix);
139:       PetscObjectSetOptionsPrefix((PetscObject)mat,mprefix);
140:       if (iscopy) {
141:         jac->tp_mat = mat;
142:       }
143:     }
144:     if (pc->pmat != pc->mat || !jac->use_true_local) {
145:       scall = MAT_INITIAL_MATRIX;
146:       if (pc->setupcalled) {
147:         if (pc->flag == SAME_NONZERO_PATTERN) {
148:           if (jac->tp_pmat) {
149:             scall = MAT_REUSE_MATRIX;
150:             pmat   = jac->tp_pmat;
151:           }
152:         } else {
153:           if (jac->tp_pmat)  {
154:             MatDestroy(jac->tp_pmat);
155:           }
156:         }
157:       }
158:       PetscObjectQueryFunction((PetscObject)pc->pmat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
159:       if (!f) {
160:         const char *type;
161:         PetscObjectGetType((PetscObject) pc->pmat,&type);
162:         SETERRQ1(PETSC_ERR_SUP,"This matrix type, %s, does not support getting diagonal block", type);
163:       }
164:       (*f)(pc->pmat,&iscopy,scall,&pmat);
165:       /* make submatrix have same prefix as entire matrix */
166:       PetscObjectGetOptionsPrefix((PetscObject)pc->pmat,&pprefix);
167:       PetscObjectSetOptionsPrefix((PetscObject)pmat,pprefix);
168:       if (iscopy) {
169:         jac->tp_pmat = pmat;
170:       }
171:     } else {
172:       pmat = mat;
173:     }
174:   }

176:   /* ------
177:      Setup code depends on the number of blocks 
178:   */
179:   if (jac->n_local == 1) {
180:     PCSetUp_BJacobi_Singleblock(pc,mat,pmat);
181:   } else {
182:     PCSetUp_BJacobi_Multiblock(pc,mat,pmat);
183:   }
184:   return(0);
185: }

187: /* Default destroy, if it has never been setup */
190: static PetscErrorCode PCDestroy_BJacobi(PC pc)
191: {
192:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;

196:   PetscFree(jac->g_lens);
197:   PetscFree(jac->l_lens);
198:   PetscFree(jac);
199:   return(0);
200: }

204: static PetscErrorCode PCSetFromOptions_BJacobi(PC pc)
205: {
206:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;
208:   PetscInt       blocks;
209:   PetscTruth     flg;

212:   PetscOptionsHead("Block Jacobi options");
213:     PetscOptionsInt("-pc_bjacobi_blocks","Total number of blocks","PCBJacobiSetTotalBlocks",jac->n,&blocks,&flg);
214:     if (flg) {
215:       PCBJacobiSetTotalBlocks(pc,blocks,PETSC_NULL);
216:     }
217:     PetscOptionsName("-pc_bjacobi_truelocal","Use the true matrix, not preconditioner matrix to define matrix vector product in sub-problems","PCBJacobiSetUseTrueLocal",&flg);
218:     if (flg) {
219:       PCBJacobiSetUseTrueLocal(pc);
220:     }
221:   PetscOptionsTail();
222:   return(0);
223: }

227: static PetscErrorCode PCView_BJacobi(PC pc,PetscViewer viewer)
228: {
229:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;
231:   PetscMPIInt    rank;
232:   PetscInt       i;
233:   PetscTruth     iascii,isstring;
234:   PetscViewer    sviewer;

237:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
238:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_STRING,&isstring);
239:   if (iascii) {
240:     if (jac->use_true_local) {
241:       PetscViewerASCIIPrintf(viewer,"  block Jacobi: using true local matrix, number of blocks = %D\n",jac->n);
242:     }
243:     PetscViewerASCIIPrintf(viewer,"  block Jacobi: number of blocks = %D\n",jac->n);
244:     MPI_Comm_rank(pc->comm,&rank);
245:     if (jac->same_local_solves) {
246:       PetscViewerASCIIPrintf(viewer,"  Local solve is same for all blocks, in the following KSP and PC objects:\n");
247:       PetscViewerGetSingleton(viewer,&sviewer);
248:       if (!rank && jac->ksp) {
249:         PetscViewerASCIIPushTab(viewer);
250:         KSPView(jac->ksp[0],sviewer);
251:         PetscViewerASCIIPopTab(viewer);
252:       }
253:       PetscViewerRestoreSingleton(viewer,&sviewer);
254:     } else {
255:       PetscInt n_global;
256:       MPI_Allreduce(&jac->n_local,&n_global,1,MPIU_INT,MPI_MAX,pc->comm);
257:       PetscViewerASCIIPrintf(viewer,"  Local solve info for each block is in the following KSP and PC objects:\n");
258:       PetscViewerASCIISynchronizedPrintf(viewer,"[%d] number of local blocks = %D, first local block number = %D\n",
259:                    rank,jac->n_local,jac->first_local);
260:       PetscViewerASCIIPushTab(viewer);
261:       for (i=0; i<n_global; i++) {
262:         PetscViewerGetSingleton(viewer,&sviewer);
263:         if (i < jac->n_local) {
264:           PetscViewerASCIISynchronizedPrintf(viewer,"[%d] local block number %D\n",rank,i);
265:           KSPView(jac->ksp[i],sviewer);
266:           PetscViewerASCIISynchronizedPrintf(viewer,"- - - - - - - - - - - - - - - - - -\n");
267:         }
268:         PetscViewerRestoreSingleton(viewer,&sviewer);
269:       }
270:       PetscViewerASCIIPopTab(viewer);
271:       PetscViewerFlush(viewer);
272:     }
273:   } else if (isstring) {
274:     PetscViewerStringSPrintf(viewer," blks=%D",jac->n);
275:     PetscViewerGetSingleton(viewer,&sviewer);
276:     if (jac->ksp) {KSPView(jac->ksp[0],sviewer);}
277:     PetscViewerRestoreSingleton(viewer,&sviewer);
278:   } else {
279:     SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for block Jacobi",((PetscObject)viewer)->type_name);
280:   }
281:   return(0);
282: }

284: /* -------------------------------------------------------------------------------------*/

289: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetUseTrueLocal_BJacobi(PC pc)
290: {
291:   PC_BJacobi   *jac;

294:   jac                 = (PC_BJacobi*)pc->data;
295:   jac->use_true_local = PETSC_TRUE;
296:   return(0);
297: }

303: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetSubKSP_BJacobi(PC pc,PetscInt *n_local,PetscInt *first_local,KSP **ksp)
304: {
305:   PC_BJacobi   *jac = (PC_BJacobi*)pc->data;;

308:   if (!pc->setupcalled) SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"Must call KSPSetUp() or PCSetUp() first");

310:   if (n_local)     *n_local     = jac->n_local;
311:   if (first_local) *first_local = jac->first_local;
312:   *ksp                          = jac->ksp;
313:   jac->same_local_solves        = PETSC_FALSE; /* Assume that local solves are now different;
314:                                                   not necessarily true though!  This flag is 
315:                                                   used only for PCView_BJacobi() */
316:   return(0);
317: }

323: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetTotalBlocks_BJacobi(PC pc,PetscInt blocks,PetscInt *lens)
324: {
325:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;


330:   if (pc->setupcalled > 0 && jac->n!=blocks) SETERRQ(PETSC_ERR_ORDER,"Cannot alter number of blocks after PCSetUp()/KSPSetUp() has been called");
331:   jac->n = blocks;
332:   if (!lens) {
333:     jac->g_lens = 0;
334:   } else {
335:     PetscMalloc(blocks*sizeof(PetscInt),&jac->g_lens);
336:     PetscLogObjectMemory(pc,blocks*sizeof(PetscInt));
337:     PetscMemcpy(jac->g_lens,lens,blocks*sizeof(PetscInt));
338:   }
339:   return(0);
340: }

346: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetTotalBlocks_BJacobi(PC pc, PetscInt *blocks, const PetscInt *lens[])
347: {
348:   PC_BJacobi *jac = (PC_BJacobi*) pc->data;

351:   *blocks = jac->n;
352:   if (lens) *lens = jac->g_lens;
353:   return(0);
354: }

360: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetLocalBlocks_BJacobi(PC pc,PetscInt blocks,const PetscInt lens[])
361: {
362:   PC_BJacobi     *jac;

366:   jac = (PC_BJacobi*)pc->data;

368:   jac->n_local = blocks;
369:   if (!lens) {
370:     jac->l_lens = 0;
371:   } else {
372:     PetscMalloc(blocks*sizeof(PetscInt),&jac->l_lens);
373:     PetscLogObjectMemory(pc,blocks*sizeof(PetscInt));
374:     PetscMemcpy(jac->l_lens,lens,blocks*sizeof(PetscInt));
375:   }
376:   return(0);
377: }

383: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetLocalBlocks_BJacobi(PC pc, PetscInt *blocks, const PetscInt *lens[])
384: {
385:   PC_BJacobi *jac = (PC_BJacobi*) pc->data;

388:   *blocks = jac->n_local;
389:   if (lens) *lens = jac->l_lens;
390:   return(0);
391: }

394: /* -------------------------------------------------------------------------------------*/

398: /*@
399:    PCBJacobiSetUseTrueLocal - Sets a flag to indicate that the block 
400:    problem is associated with the linear system matrix instead of the
401:    default (where it is associated with the preconditioning matrix).
402:    That is, if the local system is solved iteratively then it iterates
403:    on the block from the matrix using the block from the preconditioner
404:    as the preconditioner for the local block.

406:    Collective on PC

408:    Input Parameters:
409: .  pc - the preconditioner context

411:    Options Database Key:
412: .  -pc_bjacobi_truelocal - Activates PCBJacobiSetUseTrueLocal()

414:    Notes:
415:    For the common case in which the preconditioning and linear 
416:    system matrices are identical, this routine is unnecessary.

418:    Level: intermediate

420: .keywords:  block, Jacobi, set, true, local, flag

422: .seealso: PCSetOperators(), PCBJacobiSetLocalBlocks()
423: @*/
424: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetUseTrueLocal(PC pc)
425: {
426:   PetscErrorCode ierr,(*f)(PC);

430:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",(void (**)(void))&f);
431:   if (f) {
432:     (*f)(pc);
433:   }

435:   return(0);
436: }

440: /*@C
441:    PCBJacobiGetSubKSP - Gets the local KSP contexts for all blocks on
442:    this processor.
443:    
444:    Note Collective

446:    Input Parameter:
447: .  pc - the preconditioner context

449:    Output Parameters:
450: +  n_local - the number of blocks on this processor, or PETSC_NULL
451: .  first_local - the global number of the first block on this processor, or PETSC_NULL
452: -  ksp - the array of KSP contexts

454:    Notes:  
455:    After PCBJacobiGetSubKSP() the array of KSP contexts is not to be freed.
456:    
457:    Currently for some matrix implementations only 1 block per processor 
458:    is supported.
459:    
460:    You must call KSPSetUp() or PCSetUp() before calling PCBJacobiGetSubKSP().

462:    Level: advanced

464: .keywords:  block, Jacobi, get, sub, KSP, context

466: .seealso: PCBJacobiGetSubKSP()
467: @*/
468: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetSubKSP(PC pc,PetscInt *n_local,PetscInt *first_local,KSP *ksp[])
469: {
470:   PetscErrorCode ierr,(*f)(PC,PetscInt *,PetscInt *,KSP **);

474:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetSubKSP_C",(void (**)(void))&f);
475:   if (f) {
476:     (*f)(pc,n_local,first_local,ksp);
477:   } else {
478:     SETERRQ(PETSC_ERR_ARG_WRONG,"Cannot get subsolvers for this preconditioner");
479:   }
480:   return(0);
481: }

485: /*@
486:    PCBJacobiSetTotalBlocks - Sets the global number of blocks for the block
487:    Jacobi preconditioner.

489:    Collective on PC

491:    Input Parameters:
492: +  pc - the preconditioner context
493: .  blocks - the number of blocks
494: -  lens - [optional] integer array containing the size of each block

496:    Options Database Key:
497: .  -pc_bjacobi_blocks <blocks> - Sets the number of global blocks

499:    Notes:  
500:    Currently only a limited number of blocking configurations are supported.
501:    All processors sharing the PC must call this routine with the same data.

503:    Level: intermediate

505: .keywords:  set, number, Jacobi, global, total, blocks

507: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetLocalBlocks()
508: @*/
509: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetTotalBlocks(PC pc,PetscInt blocks,const PetscInt lens[])
510: {
511:   PetscErrorCode ierr,(*f)(PC,PetscInt,const PetscInt[]);

515:   if (blocks <= 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have positive blocks");
516:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetTotalBlocks_C",(void (**)(void))&f);
517:   if (f) {
518:     (*f)(pc,blocks,lens);
519:   }
520:   return(0);
521: }

525: /*@C
526:    PCBJacobiGetTotalBlocks - Gets the global number of blocks for the block
527:    Jacobi preconditioner.

529:    Collective on PC

531:    Input Parameter:
532: .  pc - the preconditioner context

534:    Output parameters:
535: +  blocks - the number of blocks
536: -  lens - integer array containing the size of each block

538:    Level: intermediate

540: .keywords:  get, number, Jacobi, global, total, blocks

542: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiGetLocalBlocks()
543: @*/
544: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetTotalBlocks(PC pc, PetscInt *blocks, const PetscInt *lens[])
545: {
546:   PetscErrorCode ierr,(*f)(PC,PetscInt*, const PetscInt *[]);

551:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetTotalBlocks_C",(void (**)(void))&f);
552:   if (f) {
553:     (*f)(pc,blocks,lens);
554:   }
555:   return(0);
556: }
557: 
560: /*@
561:    PCBJacobiSetLocalBlocks - Sets the local number of blocks for the block
562:    Jacobi preconditioner.

564:    Not Collective

566:    Input Parameters:
567: +  pc - the preconditioner context
568: .  blocks - the number of blocks
569: -  lens - [optional] integer array containing size of each block

571:    Note:  
572:    Currently only a limited number of blocking configurations are supported.

574:    Level: intermediate

576: .keywords: PC, set, number, Jacobi, local, blocks

578: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetTotalBlocks()
579: @*/
580: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetLocalBlocks(PC pc,PetscInt blocks,const PetscInt lens[])
581: {
582:   PetscErrorCode ierr,(*f)(PC,PetscInt,const PetscInt []);

586:   if (blocks < 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have nonegative blocks");
587:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetLocalBlocks_C",(void (**)(void))&f);
588:   if (f) {
589:     (*f)(pc,blocks,lens);
590:   }
591:   return(0);
592: }
593: 
596: /*@C
597:    PCBJacobiGetLocalBlocks - Gets the local number of blocks for the block
598:    Jacobi preconditioner.

600:    Not Collective

602:    Input Parameters:
603: +  pc - the preconditioner context
604: .  blocks - the number of blocks
605: -  lens - [optional] integer array containing size of each block

607:    Note:  
608:    Currently only a limited number of blocking configurations are supported.

610:    Level: intermediate

612: .keywords: PC, get, number, Jacobi, local, blocks

614: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiGetTotalBlocks()
615: @*/
616: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetLocalBlocks(PC pc, PetscInt *blocks, const PetscInt *lens[])
617: {
618:   PetscErrorCode ierr,(*f)(PC,PetscInt*, const PetscInt *[]);

623:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetLocalBlocks_C",(void (**)(void))&f);
624:   if (f) {
625:     (*f)(pc,blocks,lens);
626:   }
627:   return(0);
628: }

630: /* -----------------------------------------------------------------------------------*/

632: /*MC
633:    PCBJACOBI - Use block Jacobi preconditioning, each block is (approximately) solved with 
634:            its own KSP object.

636:    Options Database Keys:
637: .  -pc_bjacobi_truelocal - Activates PCBJacobiSetUseTrueLocal()

639:    Notes: Each processor can have one or more blocks, but a block cannot be shared by more
640:      than one processor. Defaults to one block per processor.

642:      To set options on the solvers for each block append -sub_ to all the KSP, KSP, and PC
643:         options database keys. For example, -sub_pc_type ilu -sub_pc_ilu_levels 1 -sub_ksp_type preonly
644:         
645:      To set the options on the solvers separate for each block call PCBJacobiGetSubKSP()
646:          and set the options directly on the resulting KSP object (you can access its PC
647:          KSPGetPC())

649:    Level: beginner

651:    Concepts: block Jacobi

653: .seealso:  PCCreate(), PCSetType(), PCType (for list of available types), PC,
654:            PCASM, PCBJacobiSetUseTrueLocal(), PCBJacobiGetSubKSP(), PCBJacobiSetTotalBlocks(),
655:            PCBJacobiSetLocalBlocks(), PCSetModifySubmatrices()
656: M*/

661: PetscErrorCode PETSCKSP_DLLEXPORT PCCreate_BJacobi(PC pc)
662: {
664:   PetscMPIInt    rank;
665:   PC_BJacobi     *jac;

668:   PetscNew(PC_BJacobi,&jac);
669:   PetscLogObjectMemory(pc,sizeof(PC_BJacobi));
670:   MPI_Comm_rank(pc->comm,&rank);
671:   pc->ops->apply              = 0;
672:   pc->ops->applytranspose     = 0;
673:   pc->ops->setup              = PCSetUp_BJacobi;
674:   pc->ops->destroy            = PCDestroy_BJacobi;
675:   pc->ops->setfromoptions     = PCSetFromOptions_BJacobi;
676:   pc->ops->view               = PCView_BJacobi;
677:   pc->ops->applyrichardson    = 0;

679:   pc->data               = (void*)jac;
680:   jac->n                 = -1;
681:   jac->n_local           = -1;
682:   jac->first_local       = rank;
683:   jac->ksp              = 0;
684:   jac->use_true_local    = PETSC_FALSE;
685:   jac->same_local_solves = PETSC_TRUE;
686:   jac->g_lens            = 0;
687:   jac->l_lens            = 0;
688:   jac->tp_mat            = 0;
689:   jac->tp_pmat           = 0;

691:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",
692:                     "PCBJacobiSetUseTrueLocal_BJacobi",
693:                     PCBJacobiSetUseTrueLocal_BJacobi);
694:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetSubKSP_C","PCBJacobiGetSubKSP_BJacobi",
695:                     PCBJacobiGetSubKSP_BJacobi);
696:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetTotalBlocks_C","PCBJacobiSetTotalBlocks_BJacobi",
697:                     PCBJacobiSetTotalBlocks_BJacobi);
698:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetTotalBlocks_C","PCBJacobiGetTotalBlocks_BJacobi",
699:                     PCBJacobiGetTotalBlocks_BJacobi);
700:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetLocalBlocks_C","PCBJacobiSetLocalBlocks_BJacobi",
701:                     PCBJacobiSetLocalBlocks_BJacobi);
702:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetLocalBlocks_C","PCBJacobiGetLocalBlocks_BJacobi",
703:                     PCBJacobiGetLocalBlocks_BJacobi);

705:   return(0);
706: }

709: /* --------------------------------------------------------------------------------------------*/
710: /*
711:         These are for a single block per processor; works for AIJ, BAIJ; Seq and MPI
712: */
715: PetscErrorCode PCDestroy_BJacobi_Singleblock(PC pc)
716: {
717:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
718:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
719:   PetscErrorCode         ierr;

722:   /*
723:         If the on processor block had to be generated via a MatGetDiagonalBlock()
724:      that creates a copy (for example MPIBDiag matrices do), this frees the space
725:   */
726:   if (jac->tp_mat) {
727:     MatDestroy(jac->tp_mat);
728:   }
729:   if (jac->tp_pmat) {
730:     MatDestroy(jac->tp_pmat);
731:   }

733:   KSPDestroy(jac->ksp[0]);
734:   PetscFree(jac->ksp);
735:   VecDestroy(bjac->x);
736:   VecDestroy(bjac->y);
737:   PetscFree(jac->l_lens);
738:   PetscFree(jac->g_lens);
739:   PetscFree(bjac);
740:   PetscFree(jac);
741:   return(0);
742: }

746: PetscErrorCode PCSetUpOnBlocks_BJacobi_Singleblock(PC pc)
747: {
749:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;

752:   KSPSetUp(jac->ksp[0]);
753:   return(0);
754: }

758: PetscErrorCode PCApply_BJacobi_Singleblock(PC pc,Vec x,Vec y)
759: {
760:   PetscErrorCode         ierr;
761:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
762:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
763:   PetscScalar            *x_array,*y_array;

766:   /* 
767:       The VecPlaceArray() is to avoid having to copy the 
768:     y vector into the bjac->x vector. The reason for 
769:     the bjac->x vector is that we need a sequential vector
770:     for the sequential solve.
771:   */
772:   VecGetArray(x,&x_array);
773:   VecGetArray(y,&y_array);
774:   VecPlaceArray(bjac->x,x_array);
775:   VecPlaceArray(bjac->y,y_array);
776:   KSPSolve(jac->ksp[0],bjac->x,bjac->y);
777:   VecResetArray(bjac->x);
778:   VecResetArray(bjac->y);
779:   VecRestoreArray(x,&x_array);
780:   VecRestoreArray(y,&y_array);
781:   return(0);
782: }

786: PetscErrorCode PCApplySymmetricLeft_BJacobi_Singleblock(PC pc,Vec x,Vec y)
787: {
788:   PetscErrorCode         ierr;
789:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
790:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
791:   PetscScalar            *x_array,*y_array;
792:   PC                     subpc;

795:   /* 
796:       The VecPlaceArray() is to avoid having to copy the 
797:     y vector into the bjac->x vector. The reason for 
798:     the bjac->x vector is that we need a sequential vector
799:     for the sequential solve.
800:   */
801:   VecGetArray(x,&x_array);
802:   VecGetArray(y,&y_array);
803:   VecPlaceArray(bjac->x,x_array);
804:   VecPlaceArray(bjac->y,y_array);

806:   /* apply the symmetric left portion of the inner PC operator */
807:   /* note this by-passes the inner KSP and its options completely */

809:   KSPGetPC(jac->ksp[0],&subpc);
810:   PCApplySymmetricLeft(subpc,bjac->x,bjac->y);
811:   VecResetArray(bjac->x);
812:   VecResetArray(bjac->y);

814:   VecRestoreArray(x,&x_array);
815:   VecRestoreArray(y,&y_array);
816:   return(0);
817: }

821: PetscErrorCode PCApplySymmetricRight_BJacobi_Singleblock(PC pc,Vec x,Vec y)
822: {
823:   PetscErrorCode         ierr;
824:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
825:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
826:   PetscScalar            *x_array,*y_array;
827:   PC                     subpc;

830:   /* 
831:       The VecPlaceArray() is to avoid having to copy the 
832:     y vector into the bjac->x vector. The reason for 
833:     the bjac->x vector is that we need a sequential vector
834:     for the sequential solve.
835:   */
836:   VecGetArray(x,&x_array);
837:   VecGetArray(y,&y_array);
838:   VecPlaceArray(bjac->x,x_array);
839:   VecPlaceArray(bjac->y,y_array);

841:   /* apply the symmetric right portion of the inner PC operator */
842:   /* note this by-passes the inner KSP and its options completely */

844:   KSPGetPC(jac->ksp[0],&subpc);
845:   PCApplySymmetricRight(subpc,bjac->x,bjac->y);

847:   VecRestoreArray(x,&x_array);
848:   VecRestoreArray(y,&y_array);
849:   return(0);
850: }

854: PetscErrorCode PCApplyTranspose_BJacobi_Singleblock(PC pc,Vec x,Vec y)
855: {
856:   PetscErrorCode         ierr;
857:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
858:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
859:   PetscScalar            *x_array,*y_array;

862:   /* 
863:       The VecPlaceArray() is to avoid having to copy the 
864:     y vector into the bjac->x vector. The reason for 
865:     the bjac->x vector is that we need a sequential vector
866:     for the sequential solve.
867:   */
868:   VecGetArray(x,&x_array);
869:   VecGetArray(y,&y_array);
870:   VecPlaceArray(bjac->x,x_array);
871:   VecPlaceArray(bjac->y,y_array);
872:   KSPSolveTranspose(jac->ksp[0],bjac->x,bjac->y);
873:   VecResetArray(bjac->x);
874:   VecResetArray(bjac->y);
875:   VecRestoreArray(x,&x_array);
876:   VecRestoreArray(y,&y_array);
877:   return(0);
878: }

882: static PetscErrorCode PCSetUp_BJacobi_Singleblock(PC pc,Mat mat,Mat pmat)
883: {
884:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
885:   PetscErrorCode         ierr;
886:   PetscInt               m;
887:   KSP                    ksp;
888:   Vec                    x,y;
889:   PC_BJacobi_Singleblock *bjac;
890:   PC                     subpc;
891:   PetscTruth             wasSetup;


895:   /* set default direct solver with no Krylov method */
896:   if (!pc->setupcalled) {
897:     const char *prefix;
898:     wasSetup = PETSC_FALSE;
899:     KSPCreate(PETSC_COMM_SELF,&ksp);
900:     PetscLogObjectParent(pc,ksp);
901:     KSPSetType(ksp,KSPPREONLY);
902:     KSPGetPC(ksp,&subpc);
903:     PCGetOptionsPrefix(pc,&prefix);
904:     KSPSetOptionsPrefix(ksp,prefix);
905:     KSPAppendOptionsPrefix(ksp,"sub_");
906:     /*
907:       The reason we need to generate these vectors is to serve 
908:       as the right-hand side and solution vector for the solve on the 
909:       block. We do not need to allocate space for the vectors since
910:       that is provided via VecPlaceArray() just before the call to 
911:       KSPSolve() on the block.
912:     */
913:     MatGetSize(pmat,&m,&m);
914:     VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&x);
915:     VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
916:     PetscLogObjectParent(pc,x);
917:     PetscLogObjectParent(pc,y);

919:     pc->ops->destroy             = PCDestroy_BJacobi_Singleblock;
920:     pc->ops->apply               = PCApply_BJacobi_Singleblock;
921:     pc->ops->applysymmetricleft  = PCApplySymmetricLeft_BJacobi_Singleblock;
922:     pc->ops->applysymmetricright = PCApplySymmetricRight_BJacobi_Singleblock;
923:     pc->ops->applytranspose      = PCApplyTranspose_BJacobi_Singleblock;
924:     pc->ops->setuponblocks       = PCSetUpOnBlocks_BJacobi_Singleblock;

926:     PetscMalloc(sizeof(PC_BJacobi_Singleblock),&bjac);
927:     PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Singleblock));
928:     bjac->x      = x;
929:     bjac->y      = y;

931:     PetscMalloc(sizeof(KSP),&jac->ksp);
932:     jac->ksp[0] = ksp;
933:     jac->data    = (void*)bjac;
934:   } else {
935:     wasSetup = PETSC_TRUE;
936:     ksp = jac->ksp[0];
937:     bjac = (PC_BJacobi_Singleblock *)jac->data;
938:   }
939:   if (jac->use_true_local) {
940:     KSPSetOperators(ksp,mat,pmat,pc->flag);
941:   }  else {
942:     KSPSetOperators(ksp,pmat,pmat,pc->flag);
943:   }
944:   if (!wasSetup) {
945:     KSPSetFromOptions(ksp);
946:   }
947:   return(0);
948: }

950: /* ---------------------------------------------------------------------------------------------*/

954: PetscErrorCode PCDestroy_BJacobi_Multiblock(PC pc)
955: {
956:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
957:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
958:   PetscErrorCode        ierr;
959:   PetscInt              i;

962:   MatDestroyMatrices(jac->n_local,&bjac->pmat);
963:   if (jac->use_true_local) {
964:     MatDestroyMatrices(jac->n_local,&bjac->mat);
965:   }

967:   /*
968:         If the on processor block had to be generated via a MatGetDiagonalBlock()
969:      that creates a copy (for example MPIBDiag matrices do), this frees the space
970:   */
971:   if (jac->tp_mat) {
972:     MatDestroy(jac->tp_mat);
973:   }
974:   if (jac->tp_pmat) {
975:     MatDestroy(jac->tp_pmat);
976:   }

978:   for (i=0; i<jac->n_local; i++) {
979:     KSPDestroy(jac->ksp[i]);
980:     VecDestroy(bjac->x[i]);
981:     VecDestroy(bjac->y[i]);
982:     ISDestroy(bjac->is[i]);
983:   }
984:   PetscFree(jac->ksp);
985:   PetscFree(bjac->x);
986:   PetscFree(bjac->starts);
987:   PetscFree(bjac->is);
988:   PetscFree(bjac);
989:   PetscFree(jac->l_lens);
990:   PetscFree(jac->g_lens);
991:   PetscFree(jac);
992:   return(0);
993: }

997: PetscErrorCode PCSetUpOnBlocks_BJacobi_Multiblock(PC pc)
998: {
999:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;
1001:   PetscInt       i,n_local = jac->n_local;

1004:   for (i=0; i<n_local; i++) {
1005:     KSPSetUp(jac->ksp[i]);
1006:   }
1007:   return(0);
1008: }

1010: /*
1011:       Preconditioner for block Jacobi 
1012: */
1015: PetscErrorCode PCApply_BJacobi_Multiblock(PC pc,Vec x,Vec y)
1016: {
1017:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
1018:   PetscErrorCode        ierr;
1019:   PetscInt              i,n_local = jac->n_local;
1020:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1021:   PetscScalar           *xin,*yin;
1022:   static PetscTruth     flag = PETSC_TRUE;
1023: #if defined (PETSC_USE_LOG)
1024:   static PetscEvent     SUBKspSolve;
1025: #endif
1027:   if (flag) {
1028:     PetscLogEventRegister(&SUBKspSolve,"SubKspSolve",KSP_COOKIE);
1029:     flag = PETSC_FALSE;
1030:   }
1031:   VecGetArray(x,&xin);
1032:   VecGetArray(y,&yin);
1033:   for (i=0; i<n_local; i++) {
1034:     /* 
1035:        To avoid copying the subvector from x into a workspace we instead 
1036:        make the workspace vector array point to the subpart of the array of
1037:        the global vector.
1038:     */
1039:     VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
1040:     VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);

1042:     PetscLogEventBegin(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1043:     KSPSolve(jac->ksp[i],bjac->x[i],bjac->y[i]);
1044:     PetscLogEventEnd(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);

1046:     VecResetArray(bjac->x[i]);
1047:     VecResetArray(bjac->y[i]);
1048:   }
1049:   VecRestoreArray(x,&xin);
1050:   VecRestoreArray(y,&yin);
1051:   return(0);
1052: }

1054: /*
1055:       Preconditioner for block Jacobi 
1056: */
1059: PetscErrorCode PCApplyTranspose_BJacobi_Multiblock(PC pc,Vec x,Vec y)
1060: {
1061:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
1062:   PetscErrorCode        ierr;
1063:   PetscInt              i,n_local = jac->n_local;
1064:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1065:   PetscScalar           *xin,*yin;
1066:   static PetscTruth     flag = PETSC_TRUE;
1067: #if defined (PETSC_USE_LOG)
1068:   static PetscEvent     SUBKspSolve;
1069: #endif

1072:   if (flag) {
1073:     PetscLogEventRegister(&SUBKspSolve,"SubKspSolveTranspose",KSP_COOKIE);
1074:     flag = PETSC_FALSE;
1075:   }
1076:   VecGetArray(x,&xin);
1077:   VecGetArray(y,&yin);
1078:   for (i=0; i<n_local; i++) {
1079:     /* 
1080:        To avoid copying the subvector from x into a workspace we instead 
1081:        make the workspace vector array point to the subpart of the array of
1082:        the global vector.
1083:     */
1084:     VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
1085:     VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);

1087:     PetscLogEventBegin(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1088:     KSPSolveTranspose(jac->ksp[i],bjac->x[i],bjac->y[i]);
1089:     PetscLogEventEnd(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1090:   }
1091:   VecRestoreArray(x,&xin);
1092:   VecRestoreArray(y,&yin);
1093:   return(0);
1094: }

1098: static PetscErrorCode PCSetUp_BJacobi_Multiblock(PC pc,Mat mat,Mat pmat)
1099: {
1100:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
1101:   PetscErrorCode         ierr;
1102:   PetscInt               m,n_local,N,M,start,i;
1103:   const char             *prefix,*pprefix,*mprefix;
1104:   KSP                    ksp;
1105:   Vec                    x,y;
1106:   PC_BJacobi_Multiblock  *bjac = (PC_BJacobi_Multiblock*)jac->data;
1107:   PC                     subpc;
1108:   IS                     is;
1109:   MatReuse               scall = MAT_REUSE_MATRIX;

1112:   MatGetLocalSize(pc->pmat,&M,&N);

1114:   n_local = jac->n_local;

1116:   if (jac->use_true_local) {
1117:     if (mat->type != pmat->type) SETERRQ(PETSC_ERR_ARG_INCOMP,"Matrices not of same type");
1118:   }

1120:   /* set default direct solver with no Krylov method */
1121:   if (!pc->setupcalled) {
1122:     scall                  = MAT_INITIAL_MATRIX;
1123:     pc->ops->destroy       = PCDestroy_BJacobi_Multiblock;
1124:     pc->ops->apply         = PCApply_BJacobi_Multiblock;
1125:     pc->ops->applytranspose= PCApplyTranspose_BJacobi_Multiblock;
1126:     pc->ops->setuponblocks = PCSetUpOnBlocks_BJacobi_Multiblock;

1128:     PetscMalloc(sizeof(PC_BJacobi_Multiblock),&bjac);
1129:     PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Multiblock));
1130:     PetscMalloc(n_local*sizeof(KSP),&jac->ksp);
1131:     PetscLogObjectMemory(pc,sizeof(n_local*sizeof(KSP)));
1132:     PetscMalloc(2*n_local*sizeof(Vec),&bjac->x);
1133:     PetscLogObjectMemory(pc,sizeof(2*n_local*sizeof(Vec)));
1134:     bjac->y      = bjac->x + n_local;
1135:     PetscMalloc(n_local*sizeof(PetscScalar),&bjac->starts);
1136:     PetscLogObjectMemory(pc,sizeof(n_local*sizeof(PetscScalar)));
1137: 
1138:     jac->data    = (void*)bjac;
1139:     PetscMalloc(n_local*sizeof(IS),&bjac->is);
1140:     PetscLogObjectMemory(pc,sizeof(n_local*sizeof(IS)));

1142:     start = 0;
1143:     for (i=0; i<n_local; i++) {
1144:       KSPCreate(PETSC_COMM_SELF,&ksp);
1145:       PetscLogObjectParent(pc,ksp);
1146:       KSPSetType(ksp,KSPPREONLY);
1147:       KSPGetPC(ksp,&subpc);
1148:       PCGetOptionsPrefix(pc,&prefix);
1149:       KSPSetOptionsPrefix(ksp,prefix);
1150:       KSPAppendOptionsPrefix(ksp,"sub_");

1152:       m = jac->l_lens[i];

1154:       /*
1155:       The reason we need to generate these vectors is to serve 
1156:       as the right-hand side and solution vector for the solve on the 
1157:       block. We do not need to allocate space for the vectors since
1158:       that is provided via VecPlaceArray() just before the call to 
1159:       KSPSolve() on the block.

1161:       */
1162:       VecCreateSeq(PETSC_COMM_SELF,m,&x);
1163:       VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
1164:       PetscLogObjectParent(pc,x);
1165:       PetscLogObjectParent(pc,y);
1166:       bjac->x[i]      = x;
1167:       bjac->y[i]      = y;
1168:       bjac->starts[i] = start;
1169:       jac->ksp[i]    = ksp;

1171:       ISCreateStride(PETSC_COMM_SELF,m,start,1,&is);
1172:       bjac->is[i] = is;
1173:       PetscLogObjectParent(pc,is);

1175:       start += m;
1176:     }
1177:   } else {
1178:     bjac = (PC_BJacobi_Multiblock*)jac->data;
1179:     /* 
1180:        Destroy the blocks from the previous iteration
1181:     */
1182:     if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
1183:       MatDestroyMatrices(n_local,&bjac->pmat);
1184:       if (jac->use_true_local) {
1185:         MatDestroyMatrices(n_local,&bjac->mat);
1186:       }
1187:       scall = MAT_INITIAL_MATRIX;
1188:     }
1189:   }

1191:   MatGetSubMatrices(pmat,n_local,bjac->is,bjac->is,scall,&bjac->pmat);
1192:   if (jac->use_true_local) {
1193:     PetscObjectGetOptionsPrefix((PetscObject)mat,&mprefix);
1194:     MatGetSubMatrices(mat,n_local,bjac->is,bjac->is,scall,&bjac->mat);
1195:   }
1196:   /* Return control to the user so that the submatrices can be modified (e.g., to apply
1197:      different boundary conditions for the submatrices than for the global problem) */
1198:   PCModifySubMatrices(pc,n_local,bjac->is,bjac->is,bjac->pmat,pc->modifysubmatricesP);

1200:   PetscObjectGetOptionsPrefix((PetscObject)pmat,&pprefix);
1201:   for (i=0; i<n_local; i++) {
1202:     PetscLogObjectParent(pc,bjac->pmat[i]);
1203:     PetscObjectSetOptionsPrefix((PetscObject)bjac->pmat[i],pprefix);
1204:     if (jac->use_true_local) {
1205:       PetscLogObjectParent(pc,bjac->mat[i]);
1206:       PetscObjectSetOptionsPrefix((PetscObject)bjac->mat[i],mprefix);
1207:       KSPSetOperators(jac->ksp[i],bjac->mat[i],bjac->pmat[i],pc->flag);
1208:     } else {
1209:       KSPSetOperators(jac->ksp[i],bjac->pmat[i],bjac->pmat[i],pc->flag);
1210:     }
1211:     KSPSetFromOptions(jac->ksp[i]);
1212:   }

1214:   return(0);
1215: }