Actual source code: iscoloring.c
1: #define PETSCVEC_DLL
3: #include petscsys.h
4: #include petscis.h
8: /*@
9: ISColoringDestroy - Destroys a coloring context.
11: Collective on ISColoring
13: Input Parameter:
14: . iscoloring - the coloring context
16: Level: advanced
18: .seealso: ISColoringView(), MatGetColoring()
19: @*/
20: PetscErrorCode ISColoringDestroy(ISColoring iscoloring)
21: {
22: PetscInt i;
27: if (--iscoloring->refct > 0) return(0);
29: if (iscoloring->is) {
30: for (i=0; i<iscoloring->n; i++) {
31: ISDestroy(iscoloring->is[i]);
32: }
33: PetscFree(iscoloring->is);
34: }
35: PetscFree(iscoloring->colors);
36: PetscCommDestroy(&iscoloring->comm);
37: PetscFree(iscoloring);
38: return(0);
39: }
43: /*@C
44: ISColoringView - Views a coloring context.
46: Collective on ISColoring
48: Input Parameters:
49: + iscoloring - the coloring context
50: - viewer - the viewer
52: Level: advanced
54: .seealso: ISColoringDestroy(), ISColoringGetIS(), MatGetColoring()
55: @*/
56: PetscErrorCode ISColoringView(ISColoring iscoloring,PetscViewer viewer)
57: {
58: PetscInt i;
60: PetscTruth iascii;
61: IS *is;
65: if (!viewer) viewer = PETSC_VIEWER_STDOUT_(iscoloring->comm);
68: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
69: if (iascii) {
70: MPI_Comm comm;
71: PetscMPIInt rank;
72: PetscObjectGetComm((PetscObject)viewer,&comm);
73: MPI_Comm_rank(comm,&rank);
74: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] Number of colors %d\n",rank,iscoloring->n);
75: PetscViewerFlush(viewer);
76: } else {
77: SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for ISColoring",((PetscObject)viewer)->type_name);
78: }
80: ISColoringGetIS(iscoloring,PETSC_IGNORE,&is);
81: for (i=0; i<iscoloring->n; i++) {
82: ISView(iscoloring->is[i],viewer);
83: }
84: ISColoringRestoreIS(iscoloring,&is);
85: return(0);
86: }
90: /*@C
91: ISColoringGetIS - Extracts index sets from the coloring context
93: Collective on ISColoring
95: Input Parameter:
96: . iscoloring - the coloring context
98: Output Parameters:
99: + nn - number of index sets in the coloring context
100: - is - array of index sets
102: Level: advanced
104: .seealso: ISColoringRestoreIS(), ISColoringView()
105: @*/
106: PetscErrorCode ISColoringGetIS(ISColoring iscoloring,PetscInt *nn,IS *isis[])
107: {
113: if (nn) *nn = iscoloring->n;
114: if (isis) {
115: if (!iscoloring->is) {
116: PetscInt *mcolors,**ii,nc = iscoloring->n,i,base, n = iscoloring->N;
117: ISColoringValue *colors = iscoloring->colors;
118: IS *is;
120: #if defined(PETSC_USE_DEBUG)
121: for (i=0; i<n; i++) {
122: if (((PetscInt)colors[i]) >= nc) {
123: SETERRQ3(PETSC_ERR_ARG_OUTOFRANGE,"Coloring is our of range index %d value %d number colors %d",(int)i,(int)colors[i],(int)nc);
124: }
125: }
126: #endif
127:
128: /* generate the lists of nodes for each color */
129: PetscMalloc(nc*sizeof(PetscInt),&mcolors);
130: PetscMemzero(mcolors,nc*sizeof(PetscInt));
131: for (i=0; i<n; i++) {
132: mcolors[colors[i]]++;
133: }
135: PetscMalloc(nc*sizeof(PetscInt*),&ii);
136: PetscMalloc(n*sizeof(PetscInt),&ii[0]);
137: for (i=1; i<nc; i++) {
138: ii[i] = ii[i-1] + mcolors[i-1];
139: }
140: PetscMemzero(mcolors,nc*sizeof(PetscInt));
142: if (iscoloring->ctype == IS_COLORING_LOCAL){
143: MPI_Scan(&iscoloring->N,&base,1,MPIU_INT,MPI_SUM,iscoloring->comm);
144: base -= iscoloring->N;
145: for (i=0; i<n; i++) {
146: ii[colors[i]][mcolors[colors[i]]++] = i + base; /* global idx */
147: }
148: } else if (iscoloring->ctype == IS_COLORING_GHOSTED){
149: for (i=0; i<n; i++) {
150: ii[colors[i]][mcolors[colors[i]]++] = i; /* local idx */
151: }
152: } else {
153: SETERRQ(PETSC_ERR_SUP,"Not provided for this ISColoringType type");
154: }
155:
156: PetscMalloc(nc*sizeof(IS),&is);
157: for (i=0; i<nc; i++) {
158: ISCreateGeneral(iscoloring->comm,mcolors[i],ii[i],is+i);
159: }
161: iscoloring->is = is;
162: PetscFree(ii[0]);
163: PetscFree(ii);
164: PetscFree(mcolors);
165: }
166: *isis = iscoloring->is;
167: }
168: return(0);
169: }
173: /*@C
174: ISColoringRestoreIS - Restores the index sets extracted from the coloring context
176: Collective on ISColoring
178: Input Parameter:
179: + iscoloring - the coloring context
180: - is - array of index sets
182: Level: advanced
184: .seealso: ISColoringGetIS(), ISColoringView()
185: @*/
186: PetscErrorCode ISColoringRestoreIS(ISColoring iscoloring,IS *is[])
187: {
190:
191: /* currently nothing is done here */
193: return(0);
194: }
199: /*@C
200: ISColoringCreate - Generates an ISColoring context from lists (provided
201: by each processor) of colors for each node.
203: Collective on MPI_Comm
205: Input Parameters:
206: + comm - communicator for the processors creating the coloring
207: . ncolors - max color value
208: . n - number of nodes on this processor
209: - colors - array containing the colors for this processor, color
210: numbers begin at 0. In C/C++ this array must have been obtained with PetscMalloc()
211: and should NOT be freed (The ISColoringDestroy() will free it).
213: Output Parameter:
214: . iscoloring - the resulting coloring data structure
216: Options Database Key:
217: . -is_coloring_view - Activates ISColoringView()
219: Level: advanced
220:
221: Notes: By default sets coloring type to IS_COLORING_LOCAL
223: .seealso: MatColoringCreate(), ISColoringView(), ISColoringDestroy(), ISColoringSetType()
225: @*/
226: PetscErrorCode ISColoringCreate(MPI_Comm comm,PetscInt ncolors,PetscInt n,const ISColoringValue colors[],ISColoring *iscoloring)
227: {
229: PetscMPIInt size,rank,tag;
230: PetscInt base,top,i;
231: PetscInt nc,ncwork;
232: PetscTruth flg;
233: MPI_Status status;
236: if (ncolors != PETSC_DECIDE && ncolors > IS_COLORING_MAX) {
237: if (ncolors > 65535) {
238: SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Max color value exeeds 65535 limit. This number is unrealistic. Perhaps a bug in code?\nCurrent max: %d user rewuested: %d",IS_COLORING_MAX,ncolors);
239: } else {
240: SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Max color value exeeds limit. Perhaps reconfigure PETSc with --with-is-color-value-type=short?\n Current max: %d user rewuested: %d",IS_COLORING_MAX,ncolors);
241: }
242: }
243: PetscNew(struct _n_ISColoring,iscoloring);
244: PetscCommDuplicate(comm,&(*iscoloring)->comm,&tag);
245: comm = (*iscoloring)->comm;
247: /* compute the number of the first node on my processor */
248: MPI_Comm_size(comm,&size);
250: /* should use MPI_Scan() */
251: MPI_Comm_rank(comm,&rank);
252: if (!rank) {
253: base = 0;
254: top = n;
255: } else {
256: MPI_Recv(&base,1,MPIU_INT,rank-1,tag,comm,&status);
257: top = base+n;
258: }
259: if (rank < size-1) {
260: MPI_Send(&top,1,MPIU_INT,rank+1,tag,comm);
261: }
263: /* compute the total number of colors */
264: ncwork = 0;
265: for (i=0; i<n; i++) {
266: if (ncwork < colors[i]) ncwork = colors[i];
267: }
268: ncwork++;
269: MPI_Allreduce(&ncwork,&nc,1,MPIU_INT,MPI_MAX,comm);
270: if (nc > ncolors) SETERRQ2(PETSC_ERR_ARG_INCOMP,"Number of colors passed in %D is less then the actual number of colors in array %D",ncolors,nc);
271: (*iscoloring)->n = nc;
272: (*iscoloring)->is = 0;
273: (*iscoloring)->colors = (ISColoringValue *)colors;
274: (*iscoloring)->N = n;
275: (*iscoloring)->refct = 1;
276: (*iscoloring)->ctype = IS_COLORING_LOCAL;
278: PetscOptionsHasName(PETSC_NULL,"-is_coloring_view",&flg);
279: if (flg) {
280: ISColoringView(*iscoloring,PETSC_VIEWER_STDOUT_((*iscoloring)->comm));
281: }
282: PetscInfo1(0,"Number of colors %d\n",nc);
283: return(0);
284: }
288: /*@
289: ISPartitioningToNumbering - Takes an ISPartitioning and on each processor
290: generates an IS that contains a new global node number for each index based
291: on the partitioing.
293: Collective on IS
295: Input Parameters
296: . partitioning - a partitioning as generated by MatPartitioningApply()
298: Output Parameter:
299: . is - on each processor the index set that defines the global numbers
300: (in the new numbering) for all the nodes currently (before the partitioning)
301: on that processor
303: Level: advanced
305: .seealso: MatPartitioningCreate(), AOCreateBasic(), ISPartitioningCount()
307: @*/
308: PetscErrorCode ISPartitioningToNumbering(IS part,IS *is)
309: {
310: MPI_Comm comm;
311: PetscInt i,*indices = PETSC_NULL,np,npt,n,*starts = PETSC_NULL,*sums = PETSC_NULL,*lsizes = PETSC_NULL,*newi = PETSC_NULL;
315: PetscObjectGetComm((PetscObject)part,&comm);
317: /* count the number of partitions, i.e., virtual processors */
318: ISGetLocalSize(part,&n);
319: ISGetIndices(part,&indices);
320: np = 0;
321: for (i=0; i<n; i++) {
322: np = PetscMax(np,indices[i]);
323: }
324: MPI_Allreduce(&np,&npt,1,MPIU_INT,MPI_MAX,comm);
325: np = npt+1; /* so that it looks like a MPI_Comm_size output */
327: /*
328: lsizes - number of elements of each partition on this particular processor
329: sums - total number of "previous" nodes for any particular partition
330: starts - global number of first element in each partition on this processor
331: */
332: PetscMalloc3(np,PetscInt,&lsizes,np,PetscInt,&starts,np,PetscInt,&sums);
333: PetscMemzero(lsizes,np*sizeof(PetscInt));
334: for (i=0; i<n; i++) {
335: lsizes[indices[i]]++;
336: }
337: MPI_Allreduce(lsizes,sums,np,MPIU_INT,MPI_SUM,comm);
338: MPI_Scan(lsizes,starts,np,MPIU_INT,MPI_SUM,comm);
339: for (i=0; i<np; i++) {
340: starts[i] -= lsizes[i];
341: }
342: for (i=1; i<np; i++) {
343: sums[i] += sums[i-1];
344: starts[i] += sums[i-1];
345: }
347: /*
348: For each local index give it the new global number
349: */
350: PetscMalloc(n*sizeof(PetscInt),&newi);
351: for (i=0; i<n; i++) {
352: newi[i] = starts[indices[i]]++;
353: }
354: PetscFree3(lsizes,starts,sums);
356: ISRestoreIndices(part,&indices);
357: ISCreateGeneral(comm,n,newi,is);
358: PetscFree(newi);
359: ISSetPermutation(*is);
360: return(0);
361: }
365: /*@
366: ISPartitioningCount - Takes a ISPartitioning and determines the number of
367: resulting elements on each processor
369: Collective on IS
371: Input Parameters:
372: . partitioning - a partitioning as generated by MatPartitioningApply()
374: Output Parameter:
375: . count - array of length size, to contain the number of elements assigned
376: to each partition, where size is the number of partitions generated
377: (see notes below).
379: Level: advanced
381: Notes:
382: By default the number of partitions generated (and thus the length
383: of count) is the size of the communicator associated with IS,
384: but it can be set by MatPartitioningSetNParts. The resulting array
385: of lengths can for instance serve as input of PCBJacobiSetTotalBlocks.
388: .seealso: MatPartitioningCreate(), AOCreateBasic(), ISPartitioningToNumbering(),
389: MatPartitioningSetNParts()
391: @*/
392: PetscErrorCode ISPartitioningCount(IS part,PetscInt count[])
393: {
394: MPI_Comm comm;
395: PetscInt i,*indices,np,npt,n,*lsizes;
399: PetscObjectGetComm((PetscObject)part,&comm);
401: /* count the number of partitions */
402: ISGetLocalSize(part,&n);
403: ISGetIndices(part,&indices);
404: np = 0;
405: for (i=0; i<n; i++) {
406: np = PetscMax(np,indices[i]);
407: }
408: MPI_Allreduce(&np,&npt,1,MPIU_INT,MPI_MAX,comm);
409: np = npt+1; /* so that it looks like a MPI_Comm_size output */
411: /*
412: lsizes - number of elements of each partition on this particular processor
413: sums - total number of "previous" nodes for any particular partition
414: starts - global number of first element in each partition on this processor
415: */
416: PetscMalloc(np*sizeof(PetscInt),&lsizes);
417: PetscMemzero(lsizes,np*sizeof(PetscInt));
418: for (i=0; i<n; i++) {
419: lsizes[indices[i]]++;
420: }
421: ISRestoreIndices(part,&indices);
422: MPI_Allreduce(lsizes,count,np,MPIU_INT,MPI_SUM,comm);
423: PetscFree(lsizes);
424: return(0);
425: }
429: /*@
430: ISAllGather - Given an index set (IS) on each processor, generates a large
431: index set (same on each processor) by concatenating together each
432: processors index set.
434: Collective on IS
436: Input Parameter:
437: . is - the distributed index set
439: Output Parameter:
440: . isout - the concatenated index set (same on all processors)
442: Notes:
443: ISAllGather() is clearly not scalable for large index sets.
445: The IS created on each processor must be created with a common
446: communicator (e.g., PETSC_COMM_WORLD). If the index sets were created
447: with PETSC_COMM_SELF, this routine will not work as expected, since
448: each process will generate its own new IS that consists only of
449: itself.
451: Level: intermediate
453: Concepts: gather^index sets
454: Concepts: index sets^gathering to all processors
455: Concepts: IS^gathering to all processors
457: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGatherIndices()
458: @*/
459: PetscErrorCode ISAllGather(IS is,IS *isout)
460: {
462: PetscInt *indices,n,*lindices,i,N;
463: MPI_Comm comm;
464: PetscMPIInt size,*sizes = PETSC_NULL,*offsets = PETSC_NULL,nn;
470: PetscObjectGetComm((PetscObject)is,&comm);
471: MPI_Comm_size(comm,&size);
472: PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
473:
474: ISGetLocalSize(is,&n);
475: nn = (PetscMPIInt)n;
476: MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
477: offsets[0] = 0;
478: for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
479: N = offsets[size-1] + sizes[size-1];
481: PetscMalloc(N*sizeof(PetscInt),&indices);
482: ISGetIndices(is,&lindices);
483: MPI_Allgatherv(lindices,nn,MPIU_INT,indices,sizes,offsets,MPIU_INT,comm);
484: ISRestoreIndices(is,&lindices);
485: PetscFree(sizes);
487: ISCreateGeneral(PETSC_COMM_SELF,N,indices,isout);
488: PetscFree2(indices,offsets);
489: return(0);
490: }
494: /*@C
495: ISAllGatherIndices - Given a a set of integers on each processor, generates a large
496: set (same on each processor) by concatenating together each processors integers
498: Collective on MPI_Comm
500: Input Parameter:
501: + comm - communicator to share the indices
502: . n - local size of set
503: - lindices - local indices
505: Output Parameter:
506: + outN - total number of indices
507: - outindices - all of the integers
509: Notes:
510: ISAllGatherIndices() is clearly not scalable for large index sets.
513: Level: intermediate
515: Concepts: gather^index sets
516: Concepts: index sets^gathering to all processors
517: Concepts: IS^gathering to all processors
519: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGather()
520: @*/
521: PetscErrorCode ISAllGatherIndices(MPI_Comm comm,PetscInt n,const PetscInt lindices[],PetscInt *outN,PetscInt *outindices[])
522: {
524: PetscInt *indices,i,N;
525: PetscMPIInt size,*sizes = PETSC_NULL,*offsets = PETSC_NULL,nn;
528: MPI_Comm_size(comm,&size);
529: PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
530:
531: nn = n;
532: MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
533: offsets[0] = 0;
534: for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
535: N = offsets[size-1] + sizes[size-1];
537: PetscMalloc(N*sizeof(PetscInt),&indices);
538: MPI_Allgatherv((void*)lindices,nn,MPIU_INT,indices,sizes,offsets,MPIU_INT,comm);
539: PetscFree2(sizes,offsets);
541: *outindices = indices;
542: if (outN) *outN = N;
543: return(0);
544: }
550: /*@C
551: ISAllGatherColors - Given a a set of colors on each processor, generates a large
552: set (same on each processor) by concatenating together each processors colors
554: Collective on MPI_Comm
556: Input Parameter:
557: + comm - communicator to share the indices
558: . n - local size of set
559: - lindices - local colors
561: Output Parameter:
562: + outN - total number of indices
563: - outindices - all of the colors
565: Notes:
566: ISAllGatherColors() is clearly not scalable for large index sets.
569: Level: intermediate
571: Concepts: gather^index sets
572: Concepts: index sets^gathering to all processors
573: Concepts: IS^gathering to all processors
575: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGather(), ISAllGatherIndices()
576: @*/
577: PetscErrorCode ISAllGatherColors(MPI_Comm comm,PetscInt n,ISColoringValue *lindices,PetscInt *outN,ISColoringValue *outindices[])
578: {
579: ISColoringValue *indices;
580: PetscErrorCode ierr;
581: PetscInt i,N;
582: PetscMPIInt size,*offsets = PETSC_NULL,*sizes = PETSC_NULL, nn = n;
585: MPI_Comm_size(comm,&size);
586: PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
587:
588: MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
589: offsets[0] = 0;
590: for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
591: N = offsets[size-1] + sizes[size-1];
592: PetscFree2(sizes,offsets);
594: PetscMalloc((N+1)*sizeof(ISColoringValue),&indices);
595: MPI_Allgatherv(lindices,(PetscMPIInt)n,MPIU_COLORING_VALUE,indices,sizes,offsets,MPIU_COLORING_VALUE,comm);
597: *outindices = indices;
598: if (outN) *outN = N;
599: return(0);
600: }