Actual source code: iscoloring.c

  1: #define PETSCVEC_DLL

 3:  #include petscsys.h
 4:  #include petscis.h

  8: /*@
  9:    ISColoringDestroy - Destroys a coloring context.

 11:    Collective on ISColoring

 13:    Input Parameter:
 14: .  iscoloring - the coloring context

 16:    Level: advanced

 18: .seealso: ISColoringView(), MatGetColoring()
 19: @*/
 20: PetscErrorCode PETSCVEC_DLLEXPORT ISColoringDestroy(ISColoring iscoloring)
 21: {
 22:   PetscInt i;

 27:   if (--iscoloring->refct > 0) return(0);

 29:   if (iscoloring->is) {
 30:     for (i=0; i<iscoloring->n; i++) {
 31:       ISDestroy(iscoloring->is[i]);
 32:     }
 33:     PetscFree(iscoloring->is);
 34:   }
 35:   PetscFree(iscoloring->colors);
 36:   PetscCommDestroy(&iscoloring->comm);
 37:   PetscFree(iscoloring);
 38:   return(0);
 39: }

 43: /*@C
 44:    ISColoringView - Views a coloring context.

 46:    Collective on ISColoring

 48:    Input Parameters:
 49: +  iscoloring - the coloring context
 50: -  viewer - the viewer

 52:    Level: advanced

 54: .seealso: ISColoringDestroy(), ISColoringGetIS(), MatGetColoring()
 55: @*/
 56: PetscErrorCode PETSCVEC_DLLEXPORT ISColoringView(ISColoring iscoloring,PetscViewer viewer)
 57: {
 58:   PetscInt       i;
 60:   PetscTruth     iascii;
 61:   IS             *is;

 65:   if (!viewer) viewer = PETSC_VIEWER_STDOUT_(iscoloring->comm);

 68:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
 69:   if (iascii) {
 70:     MPI_Comm    comm;
 71:     PetscMPIInt rank;
 72:     PetscObjectGetComm((PetscObject)viewer,&comm);
 73:     MPI_Comm_rank(comm,&rank);
 74:     PetscViewerASCIISynchronizedPrintf(viewer,"[%d] Number of colors %d\n",rank,iscoloring->n);
 75:     PetscViewerFlush(viewer);
 76:   } else {
 77:     SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for ISColoring",((PetscObject)viewer)->type_name);
 78:   }

 80:   ISColoringGetIS(iscoloring,PETSC_IGNORE,&is);
 81:   for (i=0; i<iscoloring->n; i++) {
 82:     ISView(iscoloring->is[i],viewer);
 83:   }
 84:   ISColoringRestoreIS(iscoloring,&is);
 85:   return(0);
 86: }

 90: /*@C
 91:    ISColoringGetIS - Extracts index sets from the coloring context

 93:    Collective on ISColoring 

 95:    Input Parameter:
 96: .  iscoloring - the coloring context

 98:    Output Parameters:
 99: +  nn - number of index sets in the coloring context
100: -  is - array of index sets

102:    Level: advanced

104: .seealso: ISColoringRestoreIS(), ISColoringView()
105: @*/
106: PetscErrorCode PETSCVEC_DLLEXPORT ISColoringGetIS(ISColoring iscoloring,PetscInt *nn,IS *isis[])
107: {


113:   if (nn)  *nn  = iscoloring->n;
114:   if (isis) {
115:     if (!iscoloring->is) {
116:       PetscInt        *mcolors,**ii,nc = iscoloring->n,i,base, n = iscoloring->N;
117:       ISColoringValue *colors = iscoloring->colors;
118:       IS              *is;

120: #if defined(PETSC_USE_DEBUG)
121:       for (i=0; i<n; i++) {
122:         if (((PetscInt)colors[i]) >= nc) {
123:           SETERRQ3(PETSC_ERR_ARG_OUTOFRANGE,"Coloring is our of range index %d value %d number colors %d",(int)i,(int)colors[i],(int)nc);
124:         }
125:       }
126: #endif
127: 
128:       /* generate the lists of nodes for each color */
129:       PetscMalloc(nc*sizeof(PetscInt),&mcolors);
130:       PetscMemzero(mcolors,nc*sizeof(PetscInt));
131:       for (i=0; i<n; i++) {
132:         mcolors[colors[i]]++;
133:       }

135:       PetscMalloc(nc*sizeof(PetscInt*),&ii);
136:       PetscMalloc(n*sizeof(PetscInt),&ii[0]);
137:       for (i=1; i<nc; i++) {
138:         ii[i] = ii[i-1] + mcolors[i-1];
139:       }
140: 
141:       MPI_Scan(&iscoloring->N,&base,1,MPIU_INT,MPI_SUM,iscoloring->comm);
142:       base -= iscoloring->N;
143:       PetscMemzero(mcolors,nc*sizeof(PetscInt));
144:       for (i=0; i<n; i++) {
145:         ii[colors[i]][mcolors[colors[i]]++] = i + base;
146:       }
147:       PetscMalloc(nc*sizeof(IS),&is);
148:       for (i=0; i<nc; i++) {
149:         ISCreateGeneral(iscoloring->comm,mcolors[i],ii[i],is+i);
150:       }

152:       iscoloring->is   = is;
153:       PetscFree(ii[0]);
154:       PetscFree(ii);
155:       PetscFree(mcolors);
156:     }
157:     *isis = iscoloring->is;
158:   }

160:   return(0);
161: }

165: /*@C
166:    ISColoringGetIS - Restores the index sets extracted from the coloring context

168:    Collective on ISColoring 

170:    Input Parameter:
171: +  iscoloring - the coloring context
172: -  is - array of index sets

174:    Level: advanced

176: .seealso: ISColoringGetIS(), ISColoringView()
177: @*/
178: PetscErrorCode PETSCVEC_DLLEXPORT ISColoringRestoreIS(ISColoring iscoloring,IS *is[])
179: {
182: 
183:   /* currently nothing is done here */

185:   return(0);
186: }


191: /*@C
192:     ISColoringCreate - Generates an ISColoring context from lists (provided 
193:     by each processor) of colors for each node.

195:     Collective on MPI_Comm

197:     Input Parameters:
198: +   comm - communicator for the processors creating the coloring
199: .   ncolors - max color value
200: .   n - number of nodes on this processor
201: -   colors - array containing the colors for this processor, color
202:              numbers begin at 0. In C/C++ this array must have been obtained with PetscMalloc()
203:              and should NOT be freed (The ISColoringDestroy() will free it).

205:     Output Parameter:
206: .   iscoloring - the resulting coloring data structure

208:     Options Database Key:
209: .   -is_coloring_view - Activates ISColoringView()

211:    Level: advanced
212:    
213:     Notes: By default sets coloring type to  IS_COLORING_LOCAL

215: .seealso: MatColoringCreate(), ISColoringView(), ISColoringDestroy(), ISColoringSetType()

217: @*/
218: PetscErrorCode PETSCVEC_DLLEXPORT ISColoringCreate(MPI_Comm comm,PetscInt ncolors,PetscInt n,const ISColoringValue colors[],ISColoring *iscoloring)
219: {
221:   PetscMPIInt    size,rank,tag;
222:   PetscInt       base,top,i;
223:   PetscInt       nc,ncwork;
224:   PetscTruth     flg;
225:   MPI_Status     status;

228:   if (ncolors != PETSC_DECIDE && ncolors > IS_COLORING_MAX) {
229:     if (ncolors > 65535) {
230:       SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Max color value exeeds 65535 limit. This number is unrealistic. Perhaps a bug in code?\nCurrent max: %d user rewuested: %d",IS_COLORING_MAX,ncolors);
231:     } else {
232:       SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Max color value exeeds limit. Perhaps reconfigure PETSc with --with-is-color-value-type=short?\n Current max: %d user rewuested: %d",IS_COLORING_MAX,ncolors);
233:     }
234:   }
235:   PetscNew(struct _n_ISColoring,iscoloring);
236:   PetscCommDuplicate(comm,&(*iscoloring)->comm,&tag);
237:   comm = (*iscoloring)->comm;

239:   /* compute the number of the first node on my processor */
240:   MPI_Comm_size(comm,&size);

242:   /* should use MPI_Scan() */
243:   MPI_Comm_rank(comm,&rank);
244:   if (!rank) {
245:     base = 0;
246:     top  = n;
247:   } else {
248:     MPI_Recv(&base,1,MPIU_INT,rank-1,tag,comm,&status);
249:     top = base+n;
250:   }
251:   if (rank < size-1) {
252:     MPI_Send(&top,1,MPIU_INT,rank+1,tag,comm);
253:   }

255:   /* compute the total number of colors */
256:   ncwork = 0;
257:   for (i=0; i<n; i++) {
258:     if (ncwork < colors[i]) ncwork = colors[i];
259:   }
260:   ncwork++;
261:   MPI_Allreduce(&ncwork,&nc,1,MPIU_INT,MPI_MAX,comm);
262:   if (nc > ncolors) SETERRQ2(PETSC_ERR_ARG_INCOMP,"Number of colors passed in %D is less then the actual number of colors in array %D",ncolors,nc);
263:   (*iscoloring)->n      = nc;
264:   (*iscoloring)->is     = 0;
265:   (*iscoloring)->colors = (ISColoringValue *)colors;
266:   (*iscoloring)->N      = n;
267:   (*iscoloring)->refct  = 1;
268:   (*iscoloring)->ctype  = IS_COLORING_LOCAL;

270:   PetscOptionsHasName(PETSC_NULL,"-is_coloring_view",&flg);
271:   if (flg) {
272:     ISColoringView(*iscoloring,PETSC_VIEWER_STDOUT_((*iscoloring)->comm));
273:   }
274:   PetscInfo1(0,"Number of colors %d\n",nc);
275:   return(0);
276: }

280: /*@
281:     ISPartitioningToNumbering - Takes an ISPartitioning and on each processor
282:     generates an IS that contains a new global node number for each index based
283:     on the partitioing.

285:     Collective on IS

287:     Input Parameters
288: .   partitioning - a partitioning as generated by MatPartitioningApply()

290:     Output Parameter:
291: .   is - on each processor the index set that defines the global numbers 
292:          (in the new numbering) for all the nodes currently (before the partitioning) 
293:          on that processor

295:    Level: advanced

297: .seealso: MatPartitioningCreate(), AOCreateBasic(), ISPartitioningCount()

299: @*/
300: PetscErrorCode PETSCVEC_DLLEXPORT ISPartitioningToNumbering(IS part,IS *is)
301: {
302:   MPI_Comm       comm;
303:   PetscInt       i,*indices,np,npt,n,*starts,*sums,*lsizes,*newi;

307:   PetscObjectGetComm((PetscObject)part,&comm);

309:   /* count the number of partitions, i.e., virtual processors */
310:   ISGetLocalSize(part,&n);
311:   ISGetIndices(part,&indices);
312:   np = 0;
313:   for (i=0; i<n; i++) {
314:     np = PetscMax(np,indices[i]);
315:   }
316:   MPI_Allreduce(&np,&npt,1,MPIU_INT,MPI_MAX,comm);
317:   np = npt+1; /* so that it looks like a MPI_Comm_size output */

319:   /*
320:         lsizes - number of elements of each partition on this particular processor
321:         sums - total number of "previous" nodes for any particular partition
322:         starts - global number of first element in each partition on this processor
323:   */
324:   PetscMalloc3(np,PetscInt,&lsizes,np,PetscInt,&starts,np,PetscInt,&sums);
325:   PetscMemzero(lsizes,np*sizeof(PetscInt));
326:   for (i=0; i<n; i++) {
327:     lsizes[indices[i]]++;
328:   }
329:   MPI_Allreduce(lsizes,sums,np,MPIU_INT,MPI_SUM,comm);
330:   MPI_Scan(lsizes,starts,np,MPIU_INT,MPI_SUM,comm);
331:   for (i=0; i<np; i++) {
332:     starts[i] -= lsizes[i];
333:   }
334:   for (i=1; i<np; i++) {
335:     sums[i]    += sums[i-1];
336:     starts[i]  += sums[i-1];
337:   }

339:   /* 
340:       For each local index give it the new global number
341:   */
342:   PetscMalloc(n*sizeof(PetscInt),&newi);
343:   for (i=0; i<n; i++) {
344:     newi[i] = starts[indices[i]]++;
345:   }
346:   PetscFree3(lsizes,starts,sums);

348:   ISRestoreIndices(part,&indices);
349:   ISCreateGeneral(comm,n,newi,is);
350:   PetscFree(newi);
351:   ISSetPermutation(*is);
352:   return(0);
353: }

357: /*@
358:     ISPartitioningCount - Takes a ISPartitioning and determines the number of 
359:     resulting elements on each processor

361:     Collective on IS

363:     Input Parameters:
364: .   partitioning - a partitioning as generated by MatPartitioningApply()

366:     Output Parameter:
367: .   count - array of length size, to contain the number of elements assigned
368:         to each partition, where size is the number of partitions generated
369:          (see notes below).

371:    Level: advanced

373:     Notes:
374:         By default the number of partitions generated (and thus the length
375:         of count) is the size of the communicator associated with IS,
376:         but it can be set by MatPartitioningSetNParts. The resulting array
377:         of lengths can for instance serve as input of PCBJacobiSetTotalBlocks.


380: .seealso: MatPartitioningCreate(), AOCreateBasic(), ISPartitioningToNumbering(),
381:         MatPartitioningSetNParts()

383: @*/
384: PetscErrorCode PETSCVEC_DLLEXPORT ISPartitioningCount(IS part,PetscInt count[])
385: {
386:   MPI_Comm       comm;
387:   PetscInt            i,*indices,np,npt,n,*lsizes;

391:   PetscObjectGetComm((PetscObject)part,&comm);

393:   /* count the number of partitions */
394:   ISGetLocalSize(part,&n);
395:   ISGetIndices(part,&indices);
396:   np = 0;
397:   for (i=0; i<n; i++) {
398:     np = PetscMax(np,indices[i]);
399:   }
400:   MPI_Allreduce(&np,&npt,1,MPIU_INT,MPI_MAX,comm);
401:   np = npt+1; /* so that it looks like a MPI_Comm_size output */

403:   /*
404:         lsizes - number of elements of each partition on this particular processor
405:         sums - total number of "previous" nodes for any particular partition
406:         starts - global number of first element in each partition on this processor
407:   */
408:   PetscMalloc(np*sizeof(PetscInt),&lsizes);
409:   PetscMemzero(lsizes,np*sizeof(PetscInt));
410:   for (i=0; i<n; i++) {
411:     lsizes[indices[i]]++;
412:   }
413:   ISRestoreIndices(part,&indices);
414:   MPI_Allreduce(lsizes,count,np,MPIU_INT,MPI_SUM,comm);
415:   PetscFree(lsizes);
416:   return(0);
417: }

421: /*@
422:     ISAllGather - Given an index set (IS) on each processor, generates a large 
423:     index set (same on each processor) by concatenating together each
424:     processors index set.

426:     Collective on IS

428:     Input Parameter:
429: .   is - the distributed index set

431:     Output Parameter:
432: .   isout - the concatenated index set (same on all processors)

434:     Notes: 
435:     ISAllGather() is clearly not scalable for large index sets.

437:     The IS created on each processor must be created with a common
438:     communicator (e.g., PETSC_COMM_WORLD). If the index sets were created 
439:     with PETSC_COMM_SELF, this routine will not work as expected, since 
440:     each process will generate its own new IS that consists only of
441:     itself.

443:     Level: intermediate

445:     Concepts: gather^index sets
446:     Concepts: index sets^gathering to all processors
447:     Concepts: IS^gathering to all processors

449: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGatherIndices()
450: @*/
451: PetscErrorCode PETSCVEC_DLLEXPORT ISAllGather(IS is,IS *isout)
452: {
454:   PetscInt       *indices,n,*lindices,i,N;
455:   MPI_Comm       comm;
456:   PetscMPIInt    size,*sizes,*offsets,nn;


462:   PetscObjectGetComm((PetscObject)is,&comm);
463:   MPI_Comm_size(comm,&size);
464:   PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
465: 
466:   ISGetLocalSize(is,&n);
467:   nn   = (PetscMPIInt)n;
468:   MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
469:   offsets[0] = 0;
470:   for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
471:   N = offsets[size-1] + sizes[size-1];

473:   PetscMalloc(N*sizeof(PetscInt),&indices);
474:   ISGetIndices(is,&lindices);
475:   MPI_Allgatherv(lindices,nn,MPIU_INT,indices,sizes,offsets,MPIU_INT,comm);
476:   ISRestoreIndices(is,&lindices);
477:   PetscFree(sizes);

479:   ISCreateGeneral(PETSC_COMM_SELF,N,indices,isout);
480:   PetscFree2(indices,offsets);
481:   return(0);
482: }

486: /*@C
487:     ISAllGatherIndices - Given a a set of integers on each processor, generates a large 
488:     set (same on each processor) by concatenating together each processors integers

490:     Collective on MPI_Comm

492:     Input Parameter:
493: +   comm - communicator to share the indices
494: .   n - local size of set
495: -   lindices - local indices

497:     Output Parameter:
498: +   outN - total number of indices
499: -   outindices - all of the integers

501:     Notes: 
502:     ISAllGatherIndices() is clearly not scalable for large index sets.


505:     Level: intermediate

507:     Concepts: gather^index sets
508:     Concepts: index sets^gathering to all processors
509:     Concepts: IS^gathering to all processors

511: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGather()
512: @*/
513: PetscErrorCode PETSCVEC_DLLEXPORT ISAllGatherIndices(MPI_Comm comm,PetscInt n,const PetscInt lindices[],PetscInt *outN,PetscInt *outindices[])
514: {
516:   PetscInt       *indices,i,N;
517:   PetscMPIInt    size,*sizes,*offsets,nn;

520:   MPI_Comm_size(comm,&size);
521:   PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
522: 
523:   nn   = n;
524:   MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
525:   offsets[0] = 0;
526:   for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
527:   N    = offsets[size-1] + sizes[size-1];

529:   PetscMalloc(N*sizeof(PetscInt),&indices);
530:   MPI_Allgatherv((void*)lindices,nn,MPIU_INT,indices,sizes,offsets,MPIU_INT,comm);
531:   PetscFree2(sizes,offsets);

533:   *outindices = indices;
534:   if (outN) *outN = N;
535:   return(0);
536: }



542: /*@C
543:     ISAllGatherColors - Given a a set of colors on each processor, generates a large 
544:     set (same on each processor) by concatenating together each processors colors

546:     Collective on MPI_Comm

548:     Input Parameter:
549: +   comm - communicator to share the indices
550: .   n - local size of set
551: -   lindices - local colors

553:     Output Parameter:
554: +   outN - total number of indices
555: -   outindices - all of the colors

557:     Notes: 
558:     ISAllGatherColors() is clearly not scalable for large index sets.


561:     Level: intermediate

563:     Concepts: gather^index sets
564:     Concepts: index sets^gathering to all processors
565:     Concepts: IS^gathering to all processors

567: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGather(), ISAllGatherIndices()
568: @*/
569: PetscErrorCode PETSCVEC_DLLEXPORT ISAllGatherColors(MPI_Comm comm,PetscInt n,ISColoringValue *lindices,PetscInt *outN,ISColoringValue *outindices[])
570: {
571:   ISColoringValue *indices;
572:   PetscErrorCode  ierr;
573:   PetscInt        i,N;
574:   PetscMPIInt     size,*offsets,*sizes, nn = n;

577:   MPI_Comm_size(comm,&size);
578:   PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
579: 
580:   MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
581:   offsets[0] = 0;
582:   for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
583:   N    = offsets[size-1] + sizes[size-1];
584:   PetscFree2(sizes,offsets);

586:   PetscMalloc((N+1)*sizeof(ISColoringValue),&indices);
587:   MPI_Allgatherv(lindices,(PetscMPIInt)n,MPIU_COLORING_VALUE,indices,sizes,offsets,MPIU_COLORING_VALUE,comm);

589:   *outindices = indices;
590:   if (outN) *outN = N;
591:   return(0);
592: }