cloudy trunk
Loading...
Searching...
No Matches
mpi_utilities.cpp
Go to the documentation of this file.
1/* This file is part of Cloudy and is copyright (C)1978-2013 by Gary J. Ferland and
2 * others. For conditions of distribution and use see copyright notice in license.txt */
3#include "cddefines.h"
4#include "mpi_utilities.h"
5#include "save.h"
6#include "dynamics.h"
7#include "grid.h"
8
9#ifndef MPI_ENABLED
10
11namespace MPI
12{
14}
15
16#endif
17
18// NB NB this routine cannot throw any exceptions as it is executed outside
19// the try{} block -- this includes mechanisms like ASSERT and cdEXIT!
20void load_balance::init(int nJobs)
21{
22 if( nJobs <= 0 )
23 return;
24
25 bool lgMPI = cpu.i().lgMPI();
26
27 p_jobs.resize( nJobs );
28 if( lgMPI )
29 p_ptr = MPI::COMM_WORLD.Get_rank();
30 else
31 p_ptr = 0;
32 // the master rank will now set up a random sequence for the jobs
33 // this way we hope to get statistical load balancing of the ranks
34 if( p_ptr == 0 )
35 {
36 for( int i=0; i < nJobs; ++i )
37 p_jobs[i] = i;
38
39 if ( lgMPI )
40 {
41 // This may or may not seed the random number generator used by
42 // random_shuffle. There is no portable C++ interface to do this :-(
43 srand( unsigned( time(NULL) ) );
44 random_shuffle( p_jobs.begin(), p_jobs.end() );
45 }
46 }
47 // now broadcast the random sequence to the other ranks...
48 if( lgMPI )
49 MPI::COMM_WORLD.Bcast( &p_jobs[0], nJobs, MPI::type(p_jobs[0]), 0 );
50}
51
52STATIC void check_grid_file( const string& fnam, int j, int ipPun );
53
56{
57 DEBUG_ENTRY( "process_output()" );
58
59 // NOTE: when this routine is called all file handles have already been closed
60
61 try
62 {
63 string main_input = save.chRedirectPrefix + ".in";
64 string main_output = save.chRedirectPrefix + ".out";
65
66 // first process main output files
67 FILE* main_output_handle = open_data( main_output.c_str(), "a", AS_LOCAL_ONLY );
68 for( int j=0; j < grid.totNumModels; ++j )
69 {
70 string Base = GridPointPrefix(j) + save.chRedirectPrefix;
71 string in_name = Base + ".in";
72 remove( in_name.c_str() );
73 string out_name = Base + ".out";
74 append_file( main_output_handle, out_name.c_str() );
75 remove( out_name.c_str() );
76 }
77 fclose( main_output_handle );
78
79 fstream main_input_handle;
80 open_data( main_input_handle, main_input.c_str(), mode_r, AS_LOCAL_ONLY );
81 string line;
82
83 int ipPun = 0;
84
85 while( getline( main_input_handle, line ) )
86 {
87 string caps_line;
88 // create all caps version
89 string::const_iterator p = line.begin();
90 while( p != line.end() )
91 caps_line.push_back( toupper(*p++) );
92 if( caps_line.compare( 0, 4, "SAVE" ) == 0 || caps_line.compare( 0, 4, "PUNC" ) == 0 )
93 {
94 ASSERT( ipPun < save.nsave );
95 string fnam = save.chFilenamePrefix;
96 string::size_type p = line.find( '"' );
97 fnam += line.substr( ++p );
98 fnam.erase( fnam.find( '"' ) );
99 // first do a minimal check on the validity of the save files
100 for( int j=0; j < grid.totNumModels; ++j )
101 check_grid_file( fnam, j, ipPun );
102 // open in binary mode in case we are writing a FITS file
103 FILE *dest = open_data( fnam.c_str(), "ab", AS_LOCAL_ONLY_TRY );
104 if( dest != NULL )
105 {
106 if( save.lgSaveToSeparateFiles[ipPun] )
107 {
108 // keep the save files for each grid point separate
109 // the main save file contains the save header
110 // salvage it by prepending it to the first save file
111 // this gives the same behavior as in non-MPI runs
112 string gridnam = GridPointPrefix(0) + fnam;
113 append_file( dest, gridnam.c_str() );
114 fclose( dest );
115 dest = NULL;
116 // this will overwrite the old file gridnam
117 rename( fnam.c_str(), gridnam.c_str() );
118 }
119 else
120 {
121 // concatenate the save files for each grid point
122 for( int j=0; j < grid.totNumModels; ++j )
123 {
124 string gridnam = GridPointPrefix(j) + fnam;
125 append_file( dest, gridnam.c_str() );
126 remove( gridnam.c_str() );
127 }
128 }
129 if( caps_line.find( "XSPE", 4 ) != string::npos )
130 {
131 // dest points to an empty file, so generate the complete FITS file now
132 ASSERT( save.FITStype[ipPun] >= 0 &&
133 save.FITStype[ipPun] < NUM_OUTPUT_TYPES );
134 saveFITSfile( dest, save.FITStype[ipPun] );
135 fseek( dest, 0, SEEK_END );
136 ASSERT( ftell(dest)%2880 == 0 );
137 }
138 if( dest != NULL )
139 {
140 fclose( dest );
141 }
142 }
143 else
144 {
145 fprintf( ioQQQ, "PROBLEM - could not open file %s\n", fnam.c_str() );
146 }
147 ++ipPun;
148 }
149 }
150 }
151 catch( ... )
152 {
153 fprintf( ioQQQ, "PROBLEM - an internal error occurred while post-processing the grid output\n" );
154 }
155}
156
158STATIC void check_grid_file( const string& fnam, int j, int ipPun )
159{
160 DEBUG_ENTRY( "check_grid_file()" );
161
162 // these are binary files, don't touch them...
163 if( save.lgFITS[ipPun] )
164 return;
165
166 bool lgForceNoDelimiter = false;
167 // in these cases there should not be a GRID_DELIMIT string...
168 if( !save.lgHashEndIter[ipPun] || !save.lg_separate_iterations[ipPun] ||
169 dynamics.lgTimeDependentStatic || strcmp( save.chHashString , "TIME_DEP" ) == 0 )
170 lgForceNoDelimiter = true;
171
172 bool lgAppendDelimiter = true;
173 bool lgAppendNewline = false;
174 string gridnam = GridPointPrefix(j) + fnam;
175 fstream str;
176 open_data( str, gridnam.c_str(), mode_r, AS_LOCAL_ONLY_TRY );
177 if( str.is_open() )
178 {
179 str.seekg( 0, ios_base::end );
180 if( str.good() && str.tellg() > 0 )
181 {
182 // check if the file ends in a newline
183 str.seekg( -1, ios_base::cur );
184 char chr;
185 str.get( chr );
186 lgAppendNewline = ( chr != '\n' );
187 // check if the GRID_DELIMIT string is present
188 string line;
189 str.seekg( 0, ios_base::beg );
190 while( getline( str, line ) )
191 {
192 if( line.find( "GRID_DELIMIT" ) != string::npos )
193 lgAppendDelimiter = false;
194 }
195 }
196 str.close();
197 }
198 if( lgForceNoDelimiter )
199 lgAppendDelimiter = false;
200 if( lgAppendNewline || lgAppendDelimiter )
201 {
202 open_data( str, gridnam.c_str(), mode_a, AS_LOCAL_ONLY_TRY );
203 if( str.is_open() )
204 {
205 if( lgAppendNewline )
206 str << endl;
207 if( lgAppendDelimiter )
208 {
209 str << save.chHashString << " GRID_DELIMIT -- grid";
210 str << setfill( '0' ) << setw(9) << j << endl;
211 }
212 str.close();
213 }
214 }
215}
216
218void append_file( FILE *dest, const char *source )
219{
220 DEBUG_ENTRY( "append_file()" );
221
222 FILE *src = open_data( source, "rb", AS_LOCAL_ONLY_TRY );
223 if( src == NULL )
224 return;
225
226 // limited testing shows that using a 4 KiB buffer should
227 // give performance that is at least very close to optimal
228 // tests were done by concatenating 10 copies of a 62.7 MiB file
229 const size_t BUF_SIZE = 4096;
230 char buf[BUF_SIZE];
231
232 while( ! feof(src) )
233 {
234 size_t nb = fread( buf, sizeof(char), BUF_SIZE, src );
235 fwrite( buf, sizeof(char), nb, dest );
236 }
237 fclose(src);
238 return;
239}
FILE * ioQQQ
Definition cddefines.cpp:7
#define ASSERT(exp)
Definition cddefines.h:578
#define STATIC
Definition cddefines.h:97
char toupper(char c)
Definition cddefines.h:700
#define DEBUG_ENTRY(funcname)
Definition cddefines.h:684
unsigned int p_ptr
vector< int > p_jobs
void init(int nJobs)
FILE * open_data(const char *fname, const char *mode, access_scheme scheme)
Definition cpu.cpp:625
static t_cpu cpu
Definition cpu.h:355
const ios_base::openmode mode_r
Definition cpu.h:212
const ios_base::openmode mode_a
Definition cpu.h:214
@ AS_LOCAL_ONLY
Definition cpu.h:208
@ AS_LOCAL_ONLY_TRY
Definition cpu.h:207
t_dynamics dynamics
Definition dynamics.cpp:44
t_grid grid
Definition grid.cpp:5
const int NUM_OUTPUT_TYPES
Definition grid.h:21
void process_output()
void append_file(FILE *dest, const char *source)
STATIC void check_grid_file(const string &fnam, int j, int ipPun)
string GridPointPrefix(int n)
t_MPI COMM_WORLD
t_save save
Definition save.cpp:5
void saveFITSfile(FILE *io, int option)
Definition save_fits.cpp:85
static double * source
Definition species2.cpp:28