Cholla 3.0.1-dev
Cholla - Massively parallel hydro on GPUs
Loading...
Searching...
No Matches
mpi_routines.h
1#ifdef MPI_CHOLLA
2 #ifndef MPI_ROUTINES_H
3 #define MPI_ROUTINES_H
4 #include <mpi.h>
5 #include <stddef.h>
6
7 #include <utility>
8
9 #include "../global/global.h"
10 #include "../grid/grid3D.h"
11
12 #ifdef FFTW
13 #include "fftw3-mpi.h"
14 #include "fftw3.h"
15 #endif /*FFTW*/
16
17/*Global MPI Variables*/
18// NOTE: some variable heavily used by mpi are declared in global.h so that they are defined even
19// when compiled without mpi
20
21extern int procID_node; /*process rank on node*/
22extern int nproc_node; /*number of MPI processes on node*/
23
24extern MPI_Comm world; /*global communicator*/
25extern MPI_Comm node; /*communicator for each node*/
26
27extern MPI_Datatype MPI_CHREAL; /*data type describing float precision*/
28
29 #ifdef PARTICLES
30extern MPI_Datatype MPI_PART_INT; /*data type describing interger for particles precision*/
31 #endif
32
33// extern MPI_Request send_request[6];
34// extern MPI_Request recv_request[6];
35extern MPI_Request *send_request;
36extern MPI_Request *recv_request;
37
38// MPI destinations and sources
39extern int dest[6];
40extern int source[6];
41
42// Communication buffers
43
44// For BLOCK
45extern Real *d_send_buffer_x0;
46extern Real *d_send_buffer_x1;
47extern Real *d_send_buffer_y0;
48extern Real *d_send_buffer_y1;
49extern Real *d_send_buffer_z0;
50extern Real *d_send_buffer_z1;
51extern Real *d_recv_buffer_x0;
52extern Real *d_recv_buffer_x1;
53extern Real *d_recv_buffer_y0;
54extern Real *d_recv_buffer_y1;
55extern Real *d_recv_buffer_z0;
56extern Real *d_recv_buffer_z1;
57
58extern Real *h_send_buffer_x0;
59extern Real *h_send_buffer_x1;
60extern Real *h_send_buffer_y0;
61extern Real *h_send_buffer_y1;
62extern Real *h_send_buffer_z0;
63extern Real *h_send_buffer_z1;
64extern Real *h_recv_buffer_x0;
65extern Real *h_recv_buffer_x1;
66extern Real *h_recv_buffer_y0;
67extern Real *h_recv_buffer_y1;
68extern Real *h_recv_buffer_z0;
69extern Real *h_recv_buffer_z1;
70
71 #ifdef PARTICLES
72// Buffers for particles transfers
73extern Real *d_send_buffer_x0_particles;
74extern Real *d_send_buffer_x1_particles;
75extern Real *d_send_buffer_y0_particles;
76extern Real *d_send_buffer_y1_particles;
77extern Real *d_send_buffer_z0_particles;
78extern Real *d_send_buffer_z1_particles;
79extern Real *d_recv_buffer_x0_particles;
80extern Real *d_recv_buffer_x1_particles;
81extern Real *d_recv_buffer_y0_particles;
82extern Real *d_recv_buffer_y1_particles;
83extern Real *d_recv_buffer_z0_particles;
84extern Real *d_recv_buffer_z1_particles;
85
86extern Real *h_send_buffer_x0_particles;
87extern Real *h_send_buffer_x1_particles;
88extern Real *h_send_buffer_y0_particles;
89extern Real *h_send_buffer_y1_particles;
90extern Real *h_send_buffer_z0_particles;
91extern Real *h_send_buffer_z1_particles;
92extern Real *h_recv_buffer_x0_particles;
93extern Real *h_recv_buffer_x1_particles;
94extern Real *h_recv_buffer_y0_particles;
95extern Real *h_recv_buffer_y1_particles;
96extern Real *h_recv_buffer_z0_particles;
97extern Real *h_recv_buffer_z1_particles;
98
99// Size of the buffers for particles transfers
100extern int buffer_length_particles_x0_send;
101extern int buffer_length_particles_x0_recv;
102extern int buffer_length_particles_x1_send;
103extern int buffer_length_particles_x1_recv;
104extern int buffer_length_particles_y0_send;
105extern int buffer_length_particles_y0_recv;
106extern int buffer_length_particles_y1_send;
107extern int buffer_length_particles_y1_recv;
108extern int buffer_length_particles_z0_send;
109extern int buffer_length_particles_z0_recv;
110extern int buffer_length_particles_z1_send;
111extern int buffer_length_particles_z1_recv;
112
113// Request for Number Of Particles to be transferred
114extern MPI_Request *send_request_n_particles;
115extern MPI_Request *recv_request_n_particles;
116// Request for Particles Transfer
117extern MPI_Request *send_request_particles_transfer;
118extern MPI_Request *recv_request_particles_transfer;
119 #endif // PARTICLES
120
121extern int send_buffer_length;
122extern int recv_buffer_length;
123extern int x_buffer_length;
124extern int y_buffer_length;
125extern int z_buffer_length;
126
127/*local domain sizes*/
128/*none of these include ghost cells!*/
129extern ptrdiff_t nx_global;
130extern ptrdiff_t ny_global;
131extern ptrdiff_t nz_global;
132extern ptrdiff_t nx_local;
133extern ptrdiff_t ny_local;
134extern ptrdiff_t nz_local;
135extern ptrdiff_t nx_local_start;
136extern ptrdiff_t ny_local_start;
137extern ptrdiff_t nz_local_start;
138
139 #ifdef FFTW
140extern ptrdiff_t n_local_complex;
141 #endif /*FFTW*/
142
143/*number of MPI procs in each dimension*/
144extern int nproc_x;
145extern int nproc_y;
146extern int nproc_z;
147
148/*\fn void InitializeChollaMPI(void) */
149/* Routine to initialize MPI */
150void InitializeChollaMPI(int *pargc, char **pargv[]);
151
152/* Perform domain decomposition */
153void DomainDecomposition(struct Parameters *P, struct Header *H, int nx_global, int ny_global, int nz_global);
154
155void DomainDecompositionBLOCK(struct Parameters *P, struct Header *H, int nx_global, int ny_global, int nz_global);
156
157/*tile MPI processes in a block decomposition*/
158void TileBlockDecomposition(void);
159
160/* MPI reduction wrapper for max(Real)*/
161Real ReduceRealMax(Real x);
162
163/* MPI reduction wrapper for min(Real)*/
164Real ReduceRealMin(Real x);
165
166/* MPI reduction wrapper for avg(Real)*/
167Real ReduceRealAvg(Real x);
168
175size_t Reduce_size_t_Max(size_t in);
176
177 #ifdef PARTICLES
178/* MPI reduction wrapper for sum(part_int)*/
179Real ReducePartIntSum(part_int_t x);
180
181// Count the particles in the MPI ranks lower that this rank to get a global
182// offset for the local IDs.
183part_int_t Get_Particles_IDs_Global_MPI_Offset(part_int_t n_local);
184
185// Function that checks if the buffer size For the particles transfer is large
186// enough, and grows the buffer if needed.
187void Check_and_Grow_Particles_Buffer(Real **part_buffer, int *current_size_ptr, int new_size);
188 #endif
189
190/* Print information about the domain properties */
191void Print_Domain_Properties(struct Header H);
192
193/* Allocate MPI communication GPU buffers for a BLOCK decomposition */
194void Allocate_MPI_DeviceBuffers(struct Header *H);
195
196/* find the greatest prime factor of an integer */
197int greatest_prime_factor(int n);
198
202int ***three_dimensional_int_array(int n, int l, int m);
203
207void deallocate_three_dimensional_int_array(int ***x, int n, int l, int m);
208
209/* Copy MPI receive buffers on Host to their device locations */
210void copyHostToDeviceReceiveBuffer(int direction);
211
217std::pair<int, int> MPI_Comm_node();
218
219 #endif /*MPI_ROUTINES_H*/
220#endif /*MPI_CHOLLA*/
Header
Definition grid3D.h:55
Parameters
Definition global.h:217
Generated by doxygen 1.9.8