Actual source code: iscoloring.c
1: #define PETSCVEC_DLL
3: #include petscsys.h
4: #include petscis.h
8: /*@
9: ISColoringDestroy - Destroys a coloring context.
11: Collective on ISColoring
13: Input Parameter:
14: . iscoloring - the coloring context
16: Level: advanced
18: .seealso: ISColoringView(), MatGetColoring()
19: @*/
20: PetscErrorCode PETSCVEC_DLLEXPORT ISColoringDestroy(ISColoring iscoloring)
21: {
22: PetscInt i;
27: if (--iscoloring->refct > 0) return(0);
29: if (iscoloring->is) {
30: for (i=0; i<iscoloring->n; i++) {
31: ISDestroy(iscoloring->is[i]);
32: }
33: PetscFree(iscoloring->is);
34: }
35: PetscFree(iscoloring->colors);
36: PetscCommDestroy(&iscoloring->comm);
37: PetscFree(iscoloring);
38: return(0);
39: }
43: /*@C
44: ISColoringView - Views a coloring context.
46: Collective on ISColoring
48: Input Parameters:
49: + iscoloring - the coloring context
50: - viewer - the viewer
52: Level: advanced
54: .seealso: ISColoringDestroy(), ISColoringGetIS(), MatGetColoring()
55: @*/
56: PetscErrorCode PETSCVEC_DLLEXPORT ISColoringView(ISColoring iscoloring,PetscViewer viewer)
57: {
58: PetscInt i;
60: PetscTruth iascii;
61: IS *is;
65: if (!viewer) viewer = PETSC_VIEWER_STDOUT_(iscoloring->comm);
68: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
69: if (iascii) {
70: MPI_Comm comm;
71: PetscMPIInt rank;
72: PetscObjectGetComm((PetscObject)viewer,&comm);
73: MPI_Comm_rank(comm,&rank);
74: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] Number of colors %d\n",rank,iscoloring->n);
75: PetscViewerFlush(viewer);
76: } else {
77: SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for ISColoring",((PetscObject)viewer)->type_name);
78: }
80: ISColoringGetIS(iscoloring,PETSC_IGNORE,&is);
81: for (i=0; i<iscoloring->n; i++) {
82: ISView(iscoloring->is[i],viewer);
83: }
84: ISColoringRestoreIS(iscoloring,&is);
85: return(0);
86: }
90: /*@C
91: ISColoringGetIS - Extracts index sets from the coloring context
93: Collective on ISColoring
95: Input Parameter:
96: . iscoloring - the coloring context
98: Output Parameters:
99: + nn - number of index sets in the coloring context
100: - is - array of index sets
102: Level: advanced
104: .seealso: ISColoringRestoreIS(), ISColoringView()
105: @*/
106: PetscErrorCode PETSCVEC_DLLEXPORT ISColoringGetIS(ISColoring iscoloring,PetscInt *nn,IS *isis[])
107: {
113: if (nn) *nn = iscoloring->n;
114: if (isis) {
115: if (!iscoloring->is) {
116: PetscInt *mcolors,**ii,nc = iscoloring->n,i,base, n = iscoloring->N;
117: ISColoringValue *colors = iscoloring->colors;
118: IS *is;
120: #if defined(PETSC_USE_DEBUG)
121: for (i=0; i<n; i++) {
122: if (((PetscInt)colors[i]) >= nc) {
123: SETERRQ3(PETSC_ERR_ARG_OUTOFRANGE,"Coloring is our of range index %d value %d number colors %d",(int)i,(int)colors[i],(int)nc);
124: }
125: }
126: #endif
127:
128: /* generate the lists of nodes for each color */
129: PetscMalloc(nc*sizeof(PetscInt),&mcolors);
130: PetscMemzero(mcolors,nc*sizeof(PetscInt));
131: for (i=0; i<n; i++) {
132: mcolors[colors[i]]++;
133: }
135: PetscMalloc(nc*sizeof(PetscInt*),&ii);
136: PetscMalloc(n*sizeof(PetscInt),&ii[0]);
137: for (i=1; i<nc; i++) {
138: ii[i] = ii[i-1] + mcolors[i-1];
139: }
140:
141: MPI_Scan(&iscoloring->N,&base,1,MPIU_INT,MPI_SUM,iscoloring->comm);
142: base -= iscoloring->N;
143: PetscMemzero(mcolors,nc*sizeof(PetscInt));
144: for (i=0; i<n; i++) {
145: ii[colors[i]][mcolors[colors[i]]++] = i + base;
146: }
147: PetscMalloc(nc*sizeof(IS),&is);
148: for (i=0; i<nc; i++) {
149: ISCreateGeneral(iscoloring->comm,mcolors[i],ii[i],is+i);
150: }
152: iscoloring->is = is;
153: PetscFree(ii[0]);
154: PetscFree(ii);
155: PetscFree(mcolors);
156: }
157: *isis = iscoloring->is;
158: }
160: return(0);
161: }
165: /*@C
166: ISColoringGetIS - Restores the index sets extracted from the coloring context
168: Collective on ISColoring
170: Input Parameter:
171: + iscoloring - the coloring context
172: - is - array of index sets
174: Level: advanced
176: .seealso: ISColoringGetIS(), ISColoringView()
177: @*/
178: PetscErrorCode PETSCVEC_DLLEXPORT ISColoringRestoreIS(ISColoring iscoloring,IS *is[])
179: {
182:
183: /* currently nothing is done here */
185: return(0);
186: }
191: /*@C
192: ISColoringCreate - Generates an ISColoring context from lists (provided
193: by each processor) of colors for each node.
195: Collective on MPI_Comm
197: Input Parameters:
198: + comm - communicator for the processors creating the coloring
199: . ncolors - max color value
200: . n - number of nodes on this processor
201: - colors - array containing the colors for this processor, color
202: numbers begin at 0. In C/C++ this array must have been obtained with PetscMalloc()
203: and should NOT be freed (The ISColoringDestroy() will free it).
205: Output Parameter:
206: . iscoloring - the resulting coloring data structure
208: Options Database Key:
209: . -is_coloring_view - Activates ISColoringView()
211: Level: advanced
212:
213: Notes: By default sets coloring type to IS_COLORING_LOCAL
215: .seealso: MatColoringCreate(), ISColoringView(), ISColoringDestroy(), ISColoringSetType()
217: @*/
218: PetscErrorCode PETSCVEC_DLLEXPORT ISColoringCreate(MPI_Comm comm,PetscInt ncolors,PetscInt n,const ISColoringValue colors[],ISColoring *iscoloring)
219: {
221: PetscMPIInt size,rank,tag;
222: PetscInt base,top,i;
223: PetscInt nc,ncwork;
224: PetscTruth flg;
225: MPI_Status status;
228: if (ncolors != PETSC_DECIDE && ncolors > IS_COLORING_MAX) {
229: if (ncolors > 65535) {
230: SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Max color value exeeds 65535 limit. This number is unrealistic. Perhaps a bug in code?\nCurrent max: %d user rewuested: %d",IS_COLORING_MAX,ncolors);
231: } else {
232: SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Max color value exeeds limit. Perhaps reconfigure PETSc with --with-is-color-value-type=short?\n Current max: %d user rewuested: %d",IS_COLORING_MAX,ncolors);
233: }
234: }
235: PetscNew(struct _n_ISColoring,iscoloring);
236: PetscCommDuplicate(comm,&(*iscoloring)->comm,&tag);
237: comm = (*iscoloring)->comm;
239: /* compute the number of the first node on my processor */
240: MPI_Comm_size(comm,&size);
242: /* should use MPI_Scan() */
243: MPI_Comm_rank(comm,&rank);
244: if (!rank) {
245: base = 0;
246: top = n;
247: } else {
248: MPI_Recv(&base,1,MPIU_INT,rank-1,tag,comm,&status);
249: top = base+n;
250: }
251: if (rank < size-1) {
252: MPI_Send(&top,1,MPIU_INT,rank+1,tag,comm);
253: }
255: /* compute the total number of colors */
256: ncwork = 0;
257: for (i=0; i<n; i++) {
258: if (ncwork < colors[i]) ncwork = colors[i];
259: }
260: ncwork++;
261: MPI_Allreduce(&ncwork,&nc,1,MPIU_INT,MPI_MAX,comm);
262: if (nc > ncolors) SETERRQ2(PETSC_ERR_ARG_INCOMP,"Number of colors passed in %D is less then the actual number of colors in array %D",ncolors,nc);
263: (*iscoloring)->n = nc;
264: (*iscoloring)->is = 0;
265: (*iscoloring)->colors = (ISColoringValue *)colors;
266: (*iscoloring)->N = n;
267: (*iscoloring)->refct = 1;
268: (*iscoloring)->ctype = IS_COLORING_LOCAL;
270: PetscOptionsHasName(PETSC_NULL,"-is_coloring_view",&flg);
271: if (flg) {
272: ISColoringView(*iscoloring,PETSC_VIEWER_STDOUT_((*iscoloring)->comm));
273: }
274: PetscInfo1(0,"Number of colors %d\n",nc);
275: return(0);
276: }
280: /*@
281: ISPartitioningToNumbering - Takes an ISPartitioning and on each processor
282: generates an IS that contains a new global node number for each index based
283: on the partitioing.
285: Collective on IS
287: Input Parameters
288: . partitioning - a partitioning as generated by MatPartitioningApply()
290: Output Parameter:
291: . is - on each processor the index set that defines the global numbers
292: (in the new numbering) for all the nodes currently (before the partitioning)
293: on that processor
295: Level: advanced
297: .seealso: MatPartitioningCreate(), AOCreateBasic(), ISPartitioningCount()
299: @*/
300: PetscErrorCode PETSCVEC_DLLEXPORT ISPartitioningToNumbering(IS part,IS *is)
301: {
302: MPI_Comm comm;
303: PetscInt i,*indices,np,npt,n,*starts,*sums,*lsizes,*newi;
307: PetscObjectGetComm((PetscObject)part,&comm);
309: /* count the number of partitions, i.e., virtual processors */
310: ISGetLocalSize(part,&n);
311: ISGetIndices(part,&indices);
312: np = 0;
313: for (i=0; i<n; i++) {
314: np = PetscMax(np,indices[i]);
315: }
316: MPI_Allreduce(&np,&npt,1,MPIU_INT,MPI_MAX,comm);
317: np = npt+1; /* so that it looks like a MPI_Comm_size output */
319: /*
320: lsizes - number of elements of each partition on this particular processor
321: sums - total number of "previous" nodes for any particular partition
322: starts - global number of first element in each partition on this processor
323: */
324: PetscMalloc3(np,PetscInt,&lsizes,np,PetscInt,&starts,np,PetscInt,&sums);
325: PetscMemzero(lsizes,np*sizeof(PetscInt));
326: for (i=0; i<n; i++) {
327: lsizes[indices[i]]++;
328: }
329: MPI_Allreduce(lsizes,sums,np,MPIU_INT,MPI_SUM,comm);
330: MPI_Scan(lsizes,starts,np,MPIU_INT,MPI_SUM,comm);
331: for (i=0; i<np; i++) {
332: starts[i] -= lsizes[i];
333: }
334: for (i=1; i<np; i++) {
335: sums[i] += sums[i-1];
336: starts[i] += sums[i-1];
337: }
339: /*
340: For each local index give it the new global number
341: */
342: PetscMalloc(n*sizeof(PetscInt),&newi);
343: for (i=0; i<n; i++) {
344: newi[i] = starts[indices[i]]++;
345: }
346: PetscFree3(lsizes,starts,sums);
348: ISRestoreIndices(part,&indices);
349: ISCreateGeneral(comm,n,newi,is);
350: PetscFree(newi);
351: ISSetPermutation(*is);
352: return(0);
353: }
357: /*@
358: ISPartitioningCount - Takes a ISPartitioning and determines the number of
359: resulting elements on each processor
361: Collective on IS
363: Input Parameters:
364: . partitioning - a partitioning as generated by MatPartitioningApply()
366: Output Parameter:
367: . count - array of length size, to contain the number of elements assigned
368: to each partition, where size is the number of partitions generated
369: (see notes below).
371: Level: advanced
373: Notes:
374: By default the number of partitions generated (and thus the length
375: of count) is the size of the communicator associated with IS,
376: but it can be set by MatPartitioningSetNParts. The resulting array
377: of lengths can for instance serve as input of PCBJacobiSetTotalBlocks.
380: .seealso: MatPartitioningCreate(), AOCreateBasic(), ISPartitioningToNumbering(),
381: MatPartitioningSetNParts()
383: @*/
384: PetscErrorCode PETSCVEC_DLLEXPORT ISPartitioningCount(IS part,PetscInt count[])
385: {
386: MPI_Comm comm;
387: PetscInt i,*indices,np,npt,n,*lsizes;
391: PetscObjectGetComm((PetscObject)part,&comm);
393: /* count the number of partitions */
394: ISGetLocalSize(part,&n);
395: ISGetIndices(part,&indices);
396: np = 0;
397: for (i=0; i<n; i++) {
398: np = PetscMax(np,indices[i]);
399: }
400: MPI_Allreduce(&np,&npt,1,MPIU_INT,MPI_MAX,comm);
401: np = npt+1; /* so that it looks like a MPI_Comm_size output */
403: /*
404: lsizes - number of elements of each partition on this particular processor
405: sums - total number of "previous" nodes for any particular partition
406: starts - global number of first element in each partition on this processor
407: */
408: PetscMalloc(np*sizeof(PetscInt),&lsizes);
409: PetscMemzero(lsizes,np*sizeof(PetscInt));
410: for (i=0; i<n; i++) {
411: lsizes[indices[i]]++;
412: }
413: ISRestoreIndices(part,&indices);
414: MPI_Allreduce(lsizes,count,np,MPIU_INT,MPI_SUM,comm);
415: PetscFree(lsizes);
416: return(0);
417: }
421: /*@
422: ISAllGather - Given an index set (IS) on each processor, generates a large
423: index set (same on each processor) by concatenating together each
424: processors index set.
426: Collective on IS
428: Input Parameter:
429: . is - the distributed index set
431: Output Parameter:
432: . isout - the concatenated index set (same on all processors)
434: Notes:
435: ISAllGather() is clearly not scalable for large index sets.
437: The IS created on each processor must be created with a common
438: communicator (e.g., PETSC_COMM_WORLD). If the index sets were created
439: with PETSC_COMM_SELF, this routine will not work as expected, since
440: each process will generate its own new IS that consists only of
441: itself.
443: Level: intermediate
445: Concepts: gather^index sets
446: Concepts: index sets^gathering to all processors
447: Concepts: IS^gathering to all processors
449: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGatherIndices()
450: @*/
451: PetscErrorCode PETSCVEC_DLLEXPORT ISAllGather(IS is,IS *isout)
452: {
454: PetscInt *indices,n,*lindices,i,N;
455: MPI_Comm comm;
456: PetscMPIInt size,*sizes,*offsets,nn;
462: PetscObjectGetComm((PetscObject)is,&comm);
463: MPI_Comm_size(comm,&size);
464: PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
465:
466: ISGetLocalSize(is,&n);
467: nn = (PetscMPIInt)n;
468: MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
469: offsets[0] = 0;
470: for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
471: N = offsets[size-1] + sizes[size-1];
473: PetscMalloc(N*sizeof(PetscInt),&indices);
474: ISGetIndices(is,&lindices);
475: MPI_Allgatherv(lindices,nn,MPIU_INT,indices,sizes,offsets,MPIU_INT,comm);
476: ISRestoreIndices(is,&lindices);
477: PetscFree(sizes);
479: ISCreateGeneral(PETSC_COMM_SELF,N,indices,isout);
480: PetscFree2(indices,offsets);
481: return(0);
482: }
486: /*@C
487: ISAllGatherIndices - Given a a set of integers on each processor, generates a large
488: set (same on each processor) by concatenating together each processors integers
490: Collective on MPI_Comm
492: Input Parameter:
493: + comm - communicator to share the indices
494: . n - local size of set
495: - lindices - local indices
497: Output Parameter:
498: + outN - total number of indices
499: - outindices - all of the integers
501: Notes:
502: ISAllGatherIndices() is clearly not scalable for large index sets.
505: Level: intermediate
507: Concepts: gather^index sets
508: Concepts: index sets^gathering to all processors
509: Concepts: IS^gathering to all processors
511: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGather()
512: @*/
513: PetscErrorCode PETSCVEC_DLLEXPORT ISAllGatherIndices(MPI_Comm comm,PetscInt n,const PetscInt lindices[],PetscInt *outN,PetscInt *outindices[])
514: {
516: PetscInt *indices,i,N;
517: PetscMPIInt size,*sizes,*offsets,nn;
520: MPI_Comm_size(comm,&size);
521: PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
522:
523: nn = n;
524: MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
525: offsets[0] = 0;
526: for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
527: N = offsets[size-1] + sizes[size-1];
529: PetscMalloc(N*sizeof(PetscInt),&indices);
530: MPI_Allgatherv((void*)lindices,nn,MPIU_INT,indices,sizes,offsets,MPIU_INT,comm);
531: PetscFree2(sizes,offsets);
533: *outindices = indices;
534: if (outN) *outN = N;
535: return(0);
536: }
542: /*@C
543: ISAllGatherColors - Given a a set of colors on each processor, generates a large
544: set (same on each processor) by concatenating together each processors colors
546: Collective on MPI_Comm
548: Input Parameter:
549: + comm - communicator to share the indices
550: . n - local size of set
551: - lindices - local colors
553: Output Parameter:
554: + outN - total number of indices
555: - outindices - all of the colors
557: Notes:
558: ISAllGatherColors() is clearly not scalable for large index sets.
561: Level: intermediate
563: Concepts: gather^index sets
564: Concepts: index sets^gathering to all processors
565: Concepts: IS^gathering to all processors
567: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGather(), ISAllGatherIndices()
568: @*/
569: PetscErrorCode PETSCVEC_DLLEXPORT ISAllGatherColors(MPI_Comm comm,PetscInt n,ISColoringValue *lindices,PetscInt *outN,ISColoringValue *outindices[])
570: {
571: ISColoringValue *indices;
572: PetscErrorCode ierr;
573: PetscInt i,N;
574: PetscMPIInt size,*offsets,*sizes, nn = n;
577: MPI_Comm_size(comm,&size);
578: PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
579:
580: MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
581: offsets[0] = 0;
582: for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
583: N = offsets[size-1] + sizes[size-1];
584: PetscFree2(sizes,offsets);
586: PetscMalloc((N+1)*sizeof(ISColoringValue),&indices);
587: MPI_Allgatherv(lindices,(PetscMPIInt)n,MPIU_COLORING_VALUE,indices,sizes,offsets,MPIU_COLORING_VALUE,comm);
589: *outindices = indices;
590: if (outN) *outN = N;
591: return(0);
592: }