Actual source code: iscoloring.c

  1: #define PETSCVEC_DLL

 3:  #include petscsys.h
 4:  #include petscis.h

  8: /*@
  9:    ISColoringDestroy - Destroys a coloring context.

 11:    Collective on ISColoring

 13:    Input Parameter:
 14: .  iscoloring - the coloring context

 16:    Level: advanced

 18: .seealso: ISColoringView(), MatGetColoring()
 19: @*/
 20: PetscErrorCode  ISColoringDestroy(ISColoring iscoloring)
 21: {
 22:   PetscInt i;

 27:   if (--iscoloring->refct > 0) return(0);

 29:   if (iscoloring->is) {
 30:     for (i=0; i<iscoloring->n; i++) {
 31:       ISDestroy(iscoloring->is[i]);
 32:     }
 33:     PetscFree(iscoloring->is);
 34:   }
 35:   PetscFree(iscoloring->colors);
 36:   PetscCommDestroy(&iscoloring->comm);
 37:   PetscFree(iscoloring);
 38:   return(0);
 39: }

 43: /*@C
 44:    ISColoringView - Views a coloring context.

 46:    Collective on ISColoring

 48:    Input Parameters:
 49: +  iscoloring - the coloring context
 50: -  viewer - the viewer

 52:    Level: advanced

 54: .seealso: ISColoringDestroy(), ISColoringGetIS(), MatGetColoring()
 55: @*/
 56: PetscErrorCode  ISColoringView(ISColoring iscoloring,PetscViewer viewer)
 57: {
 58:   PetscInt       i;
 60:   PetscTruth     iascii;
 61:   IS             *is;

 65:   if (!viewer) viewer = PETSC_VIEWER_STDOUT_(iscoloring->comm);

 68:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
 69:   if (iascii) {
 70:     MPI_Comm    comm;
 71:     PetscMPIInt rank;
 72:     PetscObjectGetComm((PetscObject)viewer,&comm);
 73:     MPI_Comm_rank(comm,&rank);
 74:     PetscViewerASCIISynchronizedPrintf(viewer,"[%d] Number of colors %d\n",rank,iscoloring->n);
 75:     PetscViewerFlush(viewer);
 76:   } else {
 77:     SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for ISColoring",((PetscObject)viewer)->type_name);
 78:   }

 80:   ISColoringGetIS(iscoloring,PETSC_IGNORE,&is);
 81:   for (i=0; i<iscoloring->n; i++) {
 82:     ISView(iscoloring->is[i],viewer);
 83:   }
 84:   ISColoringRestoreIS(iscoloring,&is);
 85:   return(0);
 86: }

 90: /*@C
 91:    ISColoringGetIS - Extracts index sets from the coloring context

 93:    Collective on ISColoring 

 95:    Input Parameter:
 96: .  iscoloring - the coloring context

 98:    Output Parameters:
 99: +  nn - number of index sets in the coloring context
100: -  is - array of index sets

102:    Level: advanced

104: .seealso: ISColoringRestoreIS(), ISColoringView()
105: @*/
106: PetscErrorCode  ISColoringGetIS(ISColoring iscoloring,PetscInt *nn,IS *isis[])
107: {


113:   if (nn)  *nn  = iscoloring->n;
114:   if (isis) {
115:     if (!iscoloring->is) {
116:       PetscInt        *mcolors,**ii,nc = iscoloring->n,i,base, n = iscoloring->N;
117:       ISColoringValue *colors = iscoloring->colors;
118:       IS              *is;

120: #if defined(PETSC_USE_DEBUG)
121:       for (i=0; i<n; i++) {
122:         if (((PetscInt)colors[i]) >= nc) {
123:           SETERRQ3(PETSC_ERR_ARG_OUTOFRANGE,"Coloring is our of range index %d value %d number colors %d",(int)i,(int)colors[i],(int)nc);
124:         }
125:       }
126: #endif
127: 
128:       /* generate the lists of nodes for each color */
129:       PetscMalloc(nc*sizeof(PetscInt),&mcolors);
130:       PetscMemzero(mcolors,nc*sizeof(PetscInt));
131:       for (i=0; i<n; i++) {
132:         mcolors[colors[i]]++;
133:       }

135:       PetscMalloc(nc*sizeof(PetscInt*),&ii);
136:       PetscMalloc(n*sizeof(PetscInt),&ii[0]);
137:       for (i=1; i<nc; i++) {
138:         ii[i] = ii[i-1] + mcolors[i-1];
139:       }
140:       PetscMemzero(mcolors,nc*sizeof(PetscInt));

142:       if (iscoloring->ctype == IS_COLORING_LOCAL){
143:         MPI_Scan(&iscoloring->N,&base,1,MPIU_INT,MPI_SUM,iscoloring->comm);
144:         base -= iscoloring->N;
145:         for (i=0; i<n; i++) {
146:           ii[colors[i]][mcolors[colors[i]]++] = i + base; /* global idx */
147:         }
148:       } else if (iscoloring->ctype == IS_COLORING_GHOSTED){
149:         for (i=0; i<n; i++) {
150:           ii[colors[i]][mcolors[colors[i]]++] = i;   /* local idx */
151:         }
152:       } else {
153:         SETERRQ(PETSC_ERR_SUP,"Not provided for this ISColoringType type");
154:       }
155: 
156:       PetscMalloc(nc*sizeof(IS),&is);
157:       for (i=0; i<nc; i++) {
158:         ISCreateGeneral(iscoloring->comm,mcolors[i],ii[i],is+i);
159:       }

161:       iscoloring->is   = is;
162:       PetscFree(ii[0]);
163:       PetscFree(ii);
164:       PetscFree(mcolors);
165:     }
166:     *isis = iscoloring->is;
167:   }
168:   return(0);
169: }

173: /*@C
174:    ISColoringRestoreIS - Restores the index sets extracted from the coloring context

176:    Collective on ISColoring 

178:    Input Parameter:
179: +  iscoloring - the coloring context
180: -  is - array of index sets

182:    Level: advanced

184: .seealso: ISColoringGetIS(), ISColoringView()
185: @*/
186: PetscErrorCode  ISColoringRestoreIS(ISColoring iscoloring,IS *is[])
187: {
190: 
191:   /* currently nothing is done here */

193:   return(0);
194: }


199: /*@C
200:     ISColoringCreate - Generates an ISColoring context from lists (provided 
201:     by each processor) of colors for each node.

203:     Collective on MPI_Comm

205:     Input Parameters:
206: +   comm - communicator for the processors creating the coloring
207: .   ncolors - max color value
208: .   n - number of nodes on this processor
209: -   colors - array containing the colors for this processor, color
210:              numbers begin at 0. In C/C++ this array must have been obtained with PetscMalloc()
211:              and should NOT be freed (The ISColoringDestroy() will free it).

213:     Output Parameter:
214: .   iscoloring - the resulting coloring data structure

216:     Options Database Key:
217: .   -is_coloring_view - Activates ISColoringView()

219:    Level: advanced
220:    
221:     Notes: By default sets coloring type to  IS_COLORING_LOCAL

223: .seealso: MatColoringCreate(), ISColoringView(), ISColoringDestroy(), ISColoringSetType()

225: @*/
226: PetscErrorCode  ISColoringCreate(MPI_Comm comm,PetscInt ncolors,PetscInt n,const ISColoringValue colors[],ISColoring *iscoloring)
227: {
229:   PetscMPIInt    size,rank,tag;
230:   PetscInt       base,top,i;
231:   PetscInt       nc,ncwork;
232:   PetscTruth     flg;
233:   MPI_Status     status;

236:   if (ncolors != PETSC_DECIDE && ncolors > IS_COLORING_MAX) {
237:     if (ncolors > 65535) {
238:       SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Max color value exeeds 65535 limit. This number is unrealistic. Perhaps a bug in code?\nCurrent max: %d user rewuested: %d",IS_COLORING_MAX,ncolors);
239:     } else {
240:       SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Max color value exeeds limit. Perhaps reconfigure PETSc with --with-is-color-value-type=short?\n Current max: %d user rewuested: %d",IS_COLORING_MAX,ncolors);
241:     }
242:   }
243:   PetscNew(struct _n_ISColoring,iscoloring);
244:   PetscCommDuplicate(comm,&(*iscoloring)->comm,&tag);
245:   comm = (*iscoloring)->comm;

247:   /* compute the number of the first node on my processor */
248:   MPI_Comm_size(comm,&size);

250:   /* should use MPI_Scan() */
251:   MPI_Comm_rank(comm,&rank);
252:   if (!rank) {
253:     base = 0;
254:     top  = n;
255:   } else {
256:     MPI_Recv(&base,1,MPIU_INT,rank-1,tag,comm,&status);
257:     top = base+n;
258:   }
259:   if (rank < size-1) {
260:     MPI_Send(&top,1,MPIU_INT,rank+1,tag,comm);
261:   }

263:   /* compute the total number of colors */
264:   ncwork = 0;
265:   for (i=0; i<n; i++) {
266:     if (ncwork < colors[i]) ncwork = colors[i];
267:   }
268:   ncwork++;
269:   MPI_Allreduce(&ncwork,&nc,1,MPIU_INT,MPI_MAX,comm);
270:   if (nc > ncolors) SETERRQ2(PETSC_ERR_ARG_INCOMP,"Number of colors passed in %D is less then the actual number of colors in array %D",ncolors,nc);
271:   (*iscoloring)->n      = nc;
272:   (*iscoloring)->is     = 0;
273:   (*iscoloring)->colors = (ISColoringValue *)colors;
274:   (*iscoloring)->N      = n;
275:   (*iscoloring)->refct  = 1;
276:   (*iscoloring)->ctype  = IS_COLORING_LOCAL;

278:   PetscOptionsHasName(PETSC_NULL,"-is_coloring_view",&flg);
279:   if (flg) {
280:     ISColoringView(*iscoloring,PETSC_VIEWER_STDOUT_((*iscoloring)->comm));
281:   }
282:   PetscInfo1(0,"Number of colors %d\n",nc);
283:   return(0);
284: }

288: /*@
289:     ISPartitioningToNumbering - Takes an ISPartitioning and on each processor
290:     generates an IS that contains a new global node number for each index based
291:     on the partitioing.

293:     Collective on IS

295:     Input Parameters
296: .   partitioning - a partitioning as generated by MatPartitioningApply()

298:     Output Parameter:
299: .   is - on each processor the index set that defines the global numbers 
300:          (in the new numbering) for all the nodes currently (before the partitioning) 
301:          on that processor

303:    Level: advanced

305: .seealso: MatPartitioningCreate(), AOCreateBasic(), ISPartitioningCount()

307: @*/
308: PetscErrorCode  ISPartitioningToNumbering(IS part,IS *is)
309: {
310:   MPI_Comm       comm;
311:   PetscInt       i,*indices = PETSC_NULL,np,npt,n,*starts = PETSC_NULL,*sums = PETSC_NULL,*lsizes = PETSC_NULL,*newi = PETSC_NULL;

315:   PetscObjectGetComm((PetscObject)part,&comm);

317:   /* count the number of partitions, i.e., virtual processors */
318:   ISGetLocalSize(part,&n);
319:   ISGetIndices(part,&indices);
320:   np = 0;
321:   for (i=0; i<n; i++) {
322:     np = PetscMax(np,indices[i]);
323:   }
324:   MPI_Allreduce(&np,&npt,1,MPIU_INT,MPI_MAX,comm);
325:   np = npt+1; /* so that it looks like a MPI_Comm_size output */

327:   /*
328:         lsizes - number of elements of each partition on this particular processor
329:         sums - total number of "previous" nodes for any particular partition
330:         starts - global number of first element in each partition on this processor
331:   */
332:   PetscMalloc3(np,PetscInt,&lsizes,np,PetscInt,&starts,np,PetscInt,&sums);
333:   PetscMemzero(lsizes,np*sizeof(PetscInt));
334:   for (i=0; i<n; i++) {
335:     lsizes[indices[i]]++;
336:   }
337:   MPI_Allreduce(lsizes,sums,np,MPIU_INT,MPI_SUM,comm);
338:   MPI_Scan(lsizes,starts,np,MPIU_INT,MPI_SUM,comm);
339:   for (i=0; i<np; i++) {
340:     starts[i] -= lsizes[i];
341:   }
342:   for (i=1; i<np; i++) {
343:     sums[i]    += sums[i-1];
344:     starts[i]  += sums[i-1];
345:   }

347:   /* 
348:       For each local index give it the new global number
349:   */
350:   PetscMalloc(n*sizeof(PetscInt),&newi);
351:   for (i=0; i<n; i++) {
352:     newi[i] = starts[indices[i]]++;
353:   }
354:   PetscFree3(lsizes,starts,sums);

356:   ISRestoreIndices(part,&indices);
357:   ISCreateGeneral(comm,n,newi,is);
358:   PetscFree(newi);
359:   ISSetPermutation(*is);
360:   return(0);
361: }

365: /*@
366:     ISPartitioningCount - Takes a ISPartitioning and determines the number of 
367:     resulting elements on each processor

369:     Collective on IS

371:     Input Parameters:
372: .   partitioning - a partitioning as generated by MatPartitioningApply()

374:     Output Parameter:
375: .   count - array of length size, to contain the number of elements assigned
376:         to each partition, where size is the number of partitions generated
377:          (see notes below).

379:    Level: advanced

381:     Notes:
382:         By default the number of partitions generated (and thus the length
383:         of count) is the size of the communicator associated with IS,
384:         but it can be set by MatPartitioningSetNParts. The resulting array
385:         of lengths can for instance serve as input of PCBJacobiSetTotalBlocks.


388: .seealso: MatPartitioningCreate(), AOCreateBasic(), ISPartitioningToNumbering(),
389:         MatPartitioningSetNParts()

391: @*/
392: PetscErrorCode  ISPartitioningCount(IS part,PetscInt count[])
393: {
394:   MPI_Comm       comm;
395:   PetscInt            i,*indices,np,npt,n,*lsizes;

399:   PetscObjectGetComm((PetscObject)part,&comm);

401:   /* count the number of partitions */
402:   ISGetLocalSize(part,&n);
403:   ISGetIndices(part,&indices);
404:   np = 0;
405:   for (i=0; i<n; i++) {
406:     np = PetscMax(np,indices[i]);
407:   }
408:   MPI_Allreduce(&np,&npt,1,MPIU_INT,MPI_MAX,comm);
409:   np = npt+1; /* so that it looks like a MPI_Comm_size output */

411:   /*
412:         lsizes - number of elements of each partition on this particular processor
413:         sums - total number of "previous" nodes for any particular partition
414:         starts - global number of first element in each partition on this processor
415:   */
416:   PetscMalloc(np*sizeof(PetscInt),&lsizes);
417:   PetscMemzero(lsizes,np*sizeof(PetscInt));
418:   for (i=0; i<n; i++) {
419:     lsizes[indices[i]]++;
420:   }
421:   ISRestoreIndices(part,&indices);
422:   MPI_Allreduce(lsizes,count,np,MPIU_INT,MPI_SUM,comm);
423:   PetscFree(lsizes);
424:   return(0);
425: }

429: /*@
430:     ISAllGather - Given an index set (IS) on each processor, generates a large 
431:     index set (same on each processor) by concatenating together each
432:     processors index set.

434:     Collective on IS

436:     Input Parameter:
437: .   is - the distributed index set

439:     Output Parameter:
440: .   isout - the concatenated index set (same on all processors)

442:     Notes: 
443:     ISAllGather() is clearly not scalable for large index sets.

445:     The IS created on each processor must be created with a common
446:     communicator (e.g., PETSC_COMM_WORLD). If the index sets were created 
447:     with PETSC_COMM_SELF, this routine will not work as expected, since 
448:     each process will generate its own new IS that consists only of
449:     itself.

451:     Level: intermediate

453:     Concepts: gather^index sets
454:     Concepts: index sets^gathering to all processors
455:     Concepts: IS^gathering to all processors

457: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGatherIndices()
458: @*/
459: PetscErrorCode  ISAllGather(IS is,IS *isout)
460: {
462:   PetscInt       *indices,n,*lindices,i,N;
463:   MPI_Comm       comm;
464:   PetscMPIInt    size,*sizes = PETSC_NULL,*offsets = PETSC_NULL,nn;


470:   PetscObjectGetComm((PetscObject)is,&comm);
471:   MPI_Comm_size(comm,&size);
472:   PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
473: 
474:   ISGetLocalSize(is,&n);
475:   nn   = (PetscMPIInt)n;
476:   MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
477:   offsets[0] = 0;
478:   for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
479:   N = offsets[size-1] + sizes[size-1];

481:   PetscMalloc(N*sizeof(PetscInt),&indices);
482:   ISGetIndices(is,&lindices);
483:   MPI_Allgatherv(lindices,nn,MPIU_INT,indices,sizes,offsets,MPIU_INT,comm);
484:   ISRestoreIndices(is,&lindices);
485:   PetscFree(sizes);

487:   ISCreateGeneral(PETSC_COMM_SELF,N,indices,isout);
488:   PetscFree2(indices,offsets);
489:   return(0);
490: }

494: /*@C
495:     ISAllGatherIndices - Given a a set of integers on each processor, generates a large 
496:     set (same on each processor) by concatenating together each processors integers

498:     Collective on MPI_Comm

500:     Input Parameter:
501: +   comm - communicator to share the indices
502: .   n - local size of set
503: -   lindices - local indices

505:     Output Parameter:
506: +   outN - total number of indices
507: -   outindices - all of the integers

509:     Notes: 
510:     ISAllGatherIndices() is clearly not scalable for large index sets.


513:     Level: intermediate

515:     Concepts: gather^index sets
516:     Concepts: index sets^gathering to all processors
517:     Concepts: IS^gathering to all processors

519: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGather()
520: @*/
521: PetscErrorCode  ISAllGatherIndices(MPI_Comm comm,PetscInt n,const PetscInt lindices[],PetscInt *outN,PetscInt *outindices[])
522: {
524:   PetscInt       *indices,i,N;
525:   PetscMPIInt    size,*sizes = PETSC_NULL,*offsets = PETSC_NULL,nn;

528:   MPI_Comm_size(comm,&size);
529:   PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
530: 
531:   nn   = n;
532:   MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
533:   offsets[0] = 0;
534:   for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
535:   N    = offsets[size-1] + sizes[size-1];

537:   PetscMalloc(N*sizeof(PetscInt),&indices);
538:   MPI_Allgatherv((void*)lindices,nn,MPIU_INT,indices,sizes,offsets,MPIU_INT,comm);
539:   PetscFree2(sizes,offsets);

541:   *outindices = indices;
542:   if (outN) *outN = N;
543:   return(0);
544: }



550: /*@C
551:     ISAllGatherColors - Given a a set of colors on each processor, generates a large 
552:     set (same on each processor) by concatenating together each processors colors

554:     Collective on MPI_Comm

556:     Input Parameter:
557: +   comm - communicator to share the indices
558: .   n - local size of set
559: -   lindices - local colors

561:     Output Parameter:
562: +   outN - total number of indices
563: -   outindices - all of the colors

565:     Notes: 
566:     ISAllGatherColors() is clearly not scalable for large index sets.


569:     Level: intermediate

571:     Concepts: gather^index sets
572:     Concepts: index sets^gathering to all processors
573:     Concepts: IS^gathering to all processors

575: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGather(), ISAllGatherIndices()
576: @*/
577: PetscErrorCode  ISAllGatherColors(MPI_Comm comm,PetscInt n,ISColoringValue *lindices,PetscInt *outN,ISColoringValue *outindices[])
578: {
579:   ISColoringValue *indices;
580:   PetscErrorCode  ierr;
581:   PetscInt        i,N;
582:   PetscMPIInt     size,*offsets = PETSC_NULL,*sizes = PETSC_NULL, nn = n;

585:   MPI_Comm_size(comm,&size);
586:   PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
587: 
588:   MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
589:   offsets[0] = 0;
590:   for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
591:   N    = offsets[size-1] + sizes[size-1];
592:   PetscFree2(sizes,offsets);

594:   PetscMalloc((N+1)*sizeof(ISColoringValue),&indices);
595:   MPI_Allgatherv(lindices,(PetscMPIInt)n,MPIU_COLORING_VALUE,indices,sizes,offsets,MPIU_COLORING_VALUE,comm);

597:   *outindices = indices;
598:   if (outN) *outN = N;
599:   return(0);
600: }