pio.h

Go to the documentation of this file.

 
#ifndef _PIO_H_
#define _PIO_H_
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <string.h> /* memcpy */
 
#ifdef MPI_SERIAL
#if defined(__cplusplus)
extern "C" {
#endif
#endif
 
#include <mpi.h>
 
#ifdef MPI_SERIAL
#if defined(__cplusplus)
}
#endif
#endif
 
#include "pio_config.h"
#if PIO_USE_PNETCDF
  #define _PNETCDF 1
#endif
#if PIO_USE_NETCDF
  #define _NETCDF 1
  #if PIO_USE_NETCDF4
    #define _NETCDF4 1
  #endif
#endif
#if PIO_USE_ADIOS
  #define _ADIOS2 1
#endif
#if PIO_USE_HDF5
  #define _HDF5 1
#endif
#if PIO_USE_MICRO_TIMING
  #define PIO_MICRO_TIMING 1
#endif
#if PIO_ENABLE_IO_STATS
  #define SPIO_IO_STATS 1
#endif
 
#ifdef _NETCDF
#include <netcdf.h>
#ifdef _NETCDF4
#include <netcdf_par.h>
#endif
#endif
#ifdef _PNETCDF
#include <pnetcdf.h>
#endif
 
#ifdef _ADIOS2
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <limits.h>
#include <adios2_c.h>
 
#define MAX_ADIOS_BUFFER_COUNT (PIO_MAX_CACHED_STEPS_FOR_ADIOS + 16) /* Maximum buffer size for aggregating decomp_id, frame_id, and fillval_id values */
#define BLOCK_MAX_BUFFER ((unsigned long)INT_MAX) /* 2GB limit of MPI_Gatherv */
/* adios end step is called if the number of blocks written out exceeds BLOCK_COUNT_THRESHOLD */
#define BLOCK_COUNT_THRESHOLD ((unsigned long)(1024 * 1024 * 1024 * 1.9))
#define BLOCK_METADATA_SIZE 70 /* Size of adios block metadata */
 
#endif
#ifdef _HDF5
#include <hdf5.h>
#include <hdf5_hl.h>
#include <unistd.h>
#endif
 
/* PIO_OFFSET_C_TYPENAME is defined in pio_config.h */
typedef PIO_OFFSET_C_TYPENAME PIO_Offset;
 
/* The MPI typename corresponding to PIO_Offset, PIO_OFFSET, is defined in pio_config.h */
 
#define PIO_IODESC_START_ID 512
#define PIO_IODESC_MAX_IDS 65536
 
#define PIO_MAX_VARS_UB 8192
#if NC_MAX_VARS > PIO_MAX_VARS_UB
#define PIO_MAX_VARS PIO_MAX_VARS_UB
#else
#define PIO_MAX_VARS NC_MAX_VARS
#endif
 
#define PIO_MAX_DIMS_UB 1024
#if NC_MAX_DIMS > PIO_MAX_DIMS_UB
#define PIO_MAX_DIMS PIO_MAX_DIMS_UB
#else
#define PIO_MAX_DIMS NC_MAX_DIMS
#endif
 
#define PIO_MAX_ATTRS_UB 8192
#if NC_MAX_ATTRS > PIO_MAX_ATTRS_UB
#define PIO_MAX_ATTRS PIO_MAX_ATTRS_UB
#else
#define PIO_MAX_ATTRS NC_MAX_ATTRS
#endif
 
#define PIO_DEFAULT (-1)
 
/* Holds the version of the decomposition file. */
#define DECOMP_VERSION_ATT_NAME "version"
 
/* Holds the maximum length of any task map. */
#define DECOMP_MAX_MAPLEN_ATT_NAME "max_maplen"
 
/* Name of title attribute. */
#define DECOMP_TITLE_ATT_NAME "title"
 
/* Name of history attribute. */
#define DECOMP_HISTORY_ATT_NAME "history"
 
/* Name of source attribute. */
#define DECOMP_SOURCE_ATT_NAME "source"
 
/* Name of array order (C or Fortran) attribute. */
#define DECOMP_ORDER_ATT_NAME "array_order"
 
/* Name of backtrace attribute. */
#define DECOMP_BACKTRACE_ATT_NAME "backtrace"
 
/* Name for the dim dim in decomp file. */
#define DECOMP_DIM_DIM "dims"
 
/* Name for the npes dim in decomp file. */
#define DECOMP_TASK_DIM_NAME "task"
 
/* Name for the npes dim in decomp file. */
#define DECOMP_MAPELEM_DIM_NAME "map_element"
 
#define DECOMP_NDIMS "ndims"
 
/* Name of var in decomp file that holds global array sizes. */
#define DECOMP_GLOBAL_SIZE_VAR_NAME "global_size"
 
/* Name of var in decomp file that holds the length of the map for
 * each task. */
#define DECOMP_MAPLEN_VAR_NAME "maplen"
 
/* Name of var in decomp file that holds map. */
#define DECOMP_MAP_VAR_NAME "map"
 
/* String used to indicate a decomposition file is in C
 * array-order. */
#define DECOMP_C_ORDER_STR "C"
 
/* String used to indicate a decomposition file is in Fortran
 * array-order. */
#define DECOMP_FORTRAN_ORDER_STR "Fortran"
 
#if defined( _PNETCDF) || defined(_NETCDF)
 
#define PIO_GLOBAL NC_GLOBAL
#define PIO_UNLIMITED NC_UNLIMITED
 
/* NetCDF types. */
#define PIO_NAT    NC_NAT
#define PIO_BYTE   NC_BYTE
#define PIO_CHAR   NC_CHAR
#define PIO_SHORT  NC_SHORT
#define PIO_INT    NC_INT
#define PIO_FLOAT  NC_FLOAT
#define PIO_REAL   NC_FLOAT
#define PIO_DOUBLE NC_DOUBLE
#define PIO_UBYTE  NC_UBYTE
#define PIO_USHORT NC_USHORT
#define PIO_UINT   NC_UINT
#define PIO_INT64  NC_INT64
#define PIO_UINT64 NC_UINT64
#define PIO_STRING NC_STRING
 
/* NetCDF flags. */
#define PIO_WRITE  NC_WRITE
#define PIO_NOWRITE  NC_NOWRITE
#define PIO_CLOBBER NC_CLOBBER
#define PIO_NOCLOBBER NC_NOCLOBBER
#define PIO_FILL NC_FILL
#define PIO_NOFILL NC_NOFILL
#define PIO_MAX_NAME_UB 1024
#if NC_MAX_NAME > PIO_MAX_NAME_UB
#define PIO_MAX_NAME PIO_MAX_NAME_UB
#else
#define PIO_MAX_NAME NC_MAX_NAME
#endif
#define PIO_MAX_VAR_DIMS_UB 1024
#if NC_MAX_VAR_DIMS > PIO_MAX_VAR_DIMS_UB
#define PIO_MAX_VAR_DIMS PIO_MAX_VAR_DIMS_UB
#else
#define PIO_MAX_VAR_DIMS NC_MAX_VAR_DIMS
#endif
#define PIO_64BIT_OFFSET NC_64BIT_OFFSET
#define PIO_64BIT_DATA NC_64BIT_DATA
 
#define PIO_NOERR  NC_NOERR
#define PIO_EBADID NC_EBADID
#define PIO_ENFILE NC_ENFILE
#define PIO_EEXIST NC_EEXIST
#define PIO_EINVAL NC_EINVAL
#define PIO_EPERM NC_EPERM
#define PIO_ENOTINDEFINE NC_ENOTINDEFINE
#define PIO_EINDEFINE NC_EINDEFINE
#define PIO_EINVALCOORDS NC_EINVALCOORDS
#define PIO_EMAXDIMS NC_EMAXDIMS
#define PIO_ENAMEINUSE NC_ENAMEINUSE
#define PIO_ENOTATT NC_ENOTATT
#define PIO_EMAXATTS NC_EMAXATTS
#define PIO_EBADTYPE NC_EBADTYPE
#define PIO_EBADDIM NC_EBADDIM
#define PIO_EUNLIMPOS NC_EUNLIMPOS
#define PIO_EMAXVARS NC_EMAXVARS
#define PIO_ENOTVAR NC_ENOTVAR
#define PIO_EGLOBAL NC_EGLOBAL
#define PIO_ENOTNC NC_ENOTNC
#define PIO_ESTS NC_ESTS
#define PIO_EMAXNAME NC_EMAXNAME
#define PIO_EUNLIMIT NC_EUNLIMIT
#define PIO_ENORECVARS NC_ENORECVARS
#define PIO_ECHAR NC_ECHAR
#define PIO_EEDGE NC_EEDGE
#define PIO_ESTRIDE NC_ESTRIDE
#define PIO_EBADNAME NC_EBADNAME
#define PIO_ERANGE NC_ERANGE
#define PIO_ENOMEM NC_ENOMEM
#define PIO_EVARSIZE NC_EVARSIZE
#define PIO_EDIMSIZE NC_EDIMSIZE
#define PIO_ETRUNC NC_ETRUNC
#define PIO_EAXISTYPE NC_EAXISTYPE
#define PIO_EDAP NC_EDAP
#define PIO_ECURL NC_ECURL
#define PIO_EIO NC_EIO
#define PIO_ENODATA NC_ENODATA
#define PIO_EDAPSVC NC_EDAPSVC
#define PIO_EDAS NC_EDAS
#define PIO_EDDS NC_EDDS
#define PIO_EDATADDS NC_EDATADDS
#define PIO_EDAPURL NC_EDAPURL
#define PIO_EDAPCONSTRAINT NC_EDAPCONSTRAINT
#define PIO_ETRANSLATION NC_ETRANSLATION
#define PIO_EHDFERR NC_EHDFERR
#define PIO_ECANTREAD NC_ECANTREAD
#define PIO_ECANTWRITE NC_ECANTWRITE
#define PIO_ECANTCREATE NC_ECANTCREATE
#define PIO_EFILEMETA NC_EFILEMETA
#define PIO_EDIMMETA NC_EDIMMETA
#define PIO_EATTMETA NC_EATTMETA
#define PIO_EVARMETA NC_EVARMETA
#define PIO_ENOCOMPOUND NC_ENOCOMPOUND
#define PIO_EATTEXISTS NC_EATTEXISTS
#define PIO_ENOTNC4 NC_ENOTNC4
#define PIO_ESTRICTNC3 NC_ESTRICTNC3
#define PIO_ENOTNC3 NC_ENOTNC3
#define PIO_ENOPAR NC_ENOPAR
#define PIO_EPARINIT NC_EPARINIT
#define PIO_EBADGRPID NC_EBADGRPID
#define PIO_EBADTYPID NC_EBADTYPID
#define PIO_ETYPDEFINED NC_ETYPDEFINED
#define PIO_EBADFIELD NC_EBADFIELD
#define PIO_EBADCLASS NC_EBADCLASS
#define PIO_EMAPTYPE NC_EMAPTYPE
#define PIO_ELATEFILL NC_ELATEFILL
#define PIO_ELATEDEF NC_ELATEDEF
#define PIO_EDIMSCALE NC_EDIMSCALE
#define PIO_ENOGRP NC_ENOGRP
#define PIO_ESTORAGE NC_ESTORAGE
#define PIO_EBADCHUNK NC_EBADCHUNK
#define PIO_ENOTBUILT NC_ENOTBUILT
#define PIO_EDISKLESS NC_EDISKLESS
 
/* These are the netCDF default fill values. */
#define PIO_FILL_BYTE NC_FILL_BYTE
#define PIO_FILL_CHAR NC_FILL_CHAR
#define PIO_FILL_SHORT NC_FILL_SHORT
#define PIO_FILL_INT NC_FILL_INT
#define PIO_FILL_FLOAT NC_FILL_FLOAT
#define PIO_FILL_DOUBLE NC_FILL_DOUBLE
#define PIO_FILL_UBYTE NC_FILL_UBYTE
#define PIO_FILL_USHORT NC_FILL_USHORT
#define PIO_FILL_UINT NC_FILL_UINT
#define PIO_FILL_INT64 NC_FILL_INT64
#define PIO_FILL_UINT64 NC_FILL_UINT64
#endif /*  defined( _PNETCDF) || defined(_NETCDF) */
 
#ifdef _PNETCDF
#define PIO_EINDEP  NC_EINDEP
#define PIO_REQ_NULL NC_REQ_NULL
#else  /* _PNETCDF */
#define PIO_EINDEP  (-203)
#define PIO_REQ_NULL (-1)
#endif /* _PNETCDF */
 
#define PIO_FIRST_ERROR_CODE (-500)
#define PIO_EBADIOTYPE  (-500)
#define PIO_EINTERNAL  (-501)
 
#ifdef _ADIOS2
#define PIO_EADIOSREAD (-300)
#define PIO_EADIOS2ERR (-301)
#endif
 
#ifdef _HDF5
#define PIO_EHDF5ERR (-700)
#endif
 
#ifdef PIO_MICRO_TIMING
typedef struct mtimer_info *mtimer_t;
#endif
 
/* Forward decl of hash map used for ADIOS reads */
struct spio_hmap;
 
typedef struct var_desc_t
{
    /* Variable ID. */
    int varid;
 
    /* Variable name - cached */
    char vname[PIO_MAX_NAME + 1];
 
    /* Variable description */
    char vdesc[PIO_MAX_NAME + 1];
    
    /* Non-zero if this is a record var (i.e. uses unlimited
     * dimension). */
    int rec_var;
 
    /* Number of dimensions for this variable - cached data */
    int ndims;
    
    int record;
 
    int *request;
 
    PIO_Offset *request_sz;
 
    int nreqs;
 
    /* Holds the fill value of this var. */
    void *fillvalue;
 
    /* The PIO data type (PIO_INT, PIO_FLOAT, etc.) */
    int pio_type;
 
    /* The size of the data type (2 for PIO_SHORT, 4 for PIO_INT, etc.) */
    PIO_Offset type_size;
 
    /* Size of one record of the variable (number of bytes)*/
    PIO_Offset vrsize;
 
    /* Bytes pending to be read */
    PIO_Offset rb_pend;
 
    /* Bytes pending to be written out */
    PIO_Offset wb_pend;
 
#ifdef PIO_MICRO_TIMING
    mtimer_t rd_mtimer;
    mtimer_t rd_rearr_mtimer;
 
    mtimer_t wr_mtimer;
    mtimer_t wr_rearr_mtimer;
#endif
 
    int use_fill;
 
    void *fillbuf;
} var_desc_t;
 
typedef struct io_region
{
    int loffset;
 
    PIO_Offset *start;
 
    PIO_Offset *count;
 
    struct io_region *next;
} io_region;
 
enum PIO_REARR_COMM_TYPE
{
    PIO_REARR_COMM_P2P = (0),
 
    PIO_REARR_COMM_COLL
};
 
enum PIO_REARR_COMM_FC_DIR
{
    PIO_REARR_COMM_FC_2D_ENABLE = (0),
 
    PIO_REARR_COMM_FC_1D_COMP2IO,
 
    PIO_REARR_COMM_FC_1D_IO2COMP,
 
    PIO_REARR_COMM_FC_2D_DISABLE
};
 
/* Constant to indicate unlimited requests. */
#define PIO_REARR_COMM_UNLIMITED_PEND_REQ -1
 
typedef struct rearr_comm_fc_opt
{
    bool hs;
 
    bool isend;
 
    int max_pend_req;
} rearr_comm_fc_opt_t;
 
typedef struct rearr_opt
{
    int comm_type;
 
    int fcd;
 
    rearr_comm_fc_opt_t comp2io;
 
    rearr_comm_fc_opt_t io2comp;
} rearr_opt_t;
 
typedef struct io_desc_t
{
    int ioid;
 
    int maplen;
 
    PIO_Offset *map;
 
    int nrecvs;
 
    int ndof;
 
    int ndims;
 
    int *dimlen;
 
    int num_aiotasks;
 
    int rearranger;
 
    int maxregions;
 
    bool needsfill;
 
    int maxbytes;
 
    int piotype;
 
    int piotype_size;
 
    MPI_Datatype mpitype;
 
    int mpitype_size;
 
    PIO_Offset llen;
 
    PIO_Offset maxiobuflen;
 
    int *rfrom;
 
    int *rcount;
 
    int *scount;
 
    PIO_Offset *sindex;
 
    PIO_Offset *rindex;
 
    MPI_Datatype *rtype;
 
    MPI_Datatype *stype;
 
    int num_stypes;
 
    int holegridsize;
 
    int maxholegridsize;
 
    int maxfillregions;
 
    io_region *firstregion;
 
    io_region *fillregion;
 
    rearr_opt_t rearr_opts;
 
    MPI_Comm subset_comm;
 
#if PIO_SAVE_DECOMPS
    /* Indicates whether this iodesc has been saved to disk (the
     * decomposition is dumped to disk)
     */
    bool is_saved;
#endif
 
    struct io_desc_t *next;
} io_desc_t;
 
/* Forward decl for I/O file summary stats info */
struct spio_io_fstats_summary;
 
typedef struct iosystem_desc_t
{
    int iosysid;
 
    /* I/O System name */
    char sname[PIO_MAX_NAME + 1];
 
    MPI_Comm union_comm;
 
    MPI_Comm io_comm;
 
    MPI_Comm comp_comm;
 
    MPI_Comm intercomm;
 
    MPI_Comm my_comm;
 
    MPI_Group compgroup;
 
    MPI_Group iogroup;
 
    int num_iotasks;
 
    int num_comptasks;
 
    int num_uniontasks;
 
    int union_rank;
 
    int comp_rank;
 
    int io_rank;
 
    int iomaster;
 
    int compmaster;
 
    int ioroot;
 
    int comproot;
 
    int *ioranks;
 
    int *compranks;
 
    int error_handler;
 
    int default_rearranger;
 
    bool async;
 
    bool ioproc;
 
    bool compproc;
 
    MPI_Info info;
 
    struct async_ios_msg_info_{
      int seq_num;
      int prev_msg;
    } async_ios_msg_info;
 
    int comp_idx;
 
    rearr_opt_t rearr_opts;
 
#ifdef _ADIOS2
    /* ADIOS handles */
    adios2_adios *adiosH; /* Handle for ADIOS write */
    adios2_adios *adios_readerH; /* Handle for ADIOS read */
    int adios_io_process;
    MPI_Comm adios_comm;
    int adios_rank, num_adiostasks;
    /* Block merging setup */
    MPI_Comm block_comm;
    int block_myrank, block_nprocs;
    #ifdef _SPIO_ADIOS_USE_COMPRESSION
    /* ADIOS operator for applying a specific lossless compression method (e.g., Blosc2, BZip2) */
    adios2_operator* lossless_compression_operator;
    int adios_lossless_compression_method;
    #ifdef _SPIO_ADIOS_USE_LOSSY_COMPRESSION
    /* ADIOS operator for applying a specific lossy compression method (e.g., SZ, MGARD, ZFP) */
    adios2_operator* lossy_compression_operator;
    int adios_lossy_compression_method;
    #endif
    #endif
#endif
 
    struct spio_io_fstats_summary *io_fstats;
 
    struct iosystem_desc_t *next;
} iosystem_desc_t;
 
typedef struct wmulti_buffer
{
    int ioid;
 
    int recordvar;
 
    int num_arrays;
 
    int arraylen;
 
    int *vid;
 
    int *frame;
 
    void *fillvalue;
 
    void *data;
 
    struct wmulti_buffer *next;
} wmulti_buffer;
 
#ifdef _ADIOS2
/* Interval map for ADIOS read */
typedef struct adios_interval_map_t
{
    int n_adios_steps;
    short **map;
} adios_interval_map_t;
 
/* Maximum char array length to store "darray" or "put_var", for ADIOS read */
#define NC_OP_TAG_MAX_LEN 8
 
typedef struct adios_var_desc_t
{
    char * name;
 
    int nc_type;
 
    adios2_type adios_type;
    size_t adios_type_size;
 
    int ndims;
 
    int * gdimids;
 
    int nattrs;
 
    adios2_variable* adios_varid; // 0: undefined yet
 
    /* to handle PIOc_setframe with different decompositions */
    adios2_variable* decomp_varid;
    adios2_variable* frame_varid;
    adios2_variable* fillval_varid;
    adios2_variable* num_block_writers_varid;
 
    /* to handle multi-dimensional temporal variables */
    adios2_variable* start_varid;
    adios2_variable* count_varid;
 
    /* to buffer decomp id, frame id, fill value, and writer blocks */
    int32_t *decomp_buffer;
    int32_t *frame_buffer;
    char *fillval_buffer;
    int32_t fillval_size;
    int32_t *num_wb_buffer;
    int32_t decomp_cnt, frame_cnt, fillval_cnt, num_wb_cnt;
    int32_t max_buffer_cnt;
 
    /* Simplified version of an interval map implementation
     * index is a frame_id and the value is the adios_step */
    adios_interval_map_t* interval_map; /* For ADIOS read */
 
    /* Is this variable written with put_var or darray? */
    char nc_op_tag[NC_OP_TAG_MAX_LEN]; /* For ADIOS read */
} adios_var_desc_t;
 
/* Track attributes */
typedef struct adios_att_desc_t
{
    char *att_name;
 
    nc_type att_type;
 
    PIO_Offset att_len;
 
    int att_ncid;
 
    int att_varid;
 
    adios2_type adios_type;
} adios_att_desc_t;
 
#ifdef _SPIO_ADIOS_USE_COMPRESSION
enum ADIOS_COMPRESSION_METHOD
{
    ADIOS_COMPRESSION_METHOD_BLOSC2 = 1,
 
    ADIOS_COMPRESSION_METHOD_BZIP2 = 2,
 
    ADIOS_COMPRESSION_METHOD_MGARD = 3,
 
    ADIOS_COMPRESSION_METHOD_SZ = 4,
 
    ADIOS_COMPRESSION_METHOD_ZFP = 5
};
#endif
#endif /* _ADIOS2 */
 
#ifdef _HDF5
typedef struct hdf5_dim_desc_t
{
    char* name;
 
    PIO_Offset len;
 
    bool has_coord_var;
 
    hid_t hdf5_dataset_id;
} hdf5_dim_desc_t;
 
typedef struct hdf5_var_desc_t
{
    char* name;
 
    /* Alternative name for a non-coordinate variable with the same name as a dimension */
    char* alt_name;
 
    int nc_type;
 
    hid_t hdf5_type;
 
    int ndims;
 
    int* hdf5_dimids;
 
    bool is_coord_var;
 
    hid_t hdf5_dataset_id;
} hdf5_var_desc_t;
 
typedef struct hdf5_att_desc_t
{
    char *att_name;
 
    nc_type att_type;
 
    PIO_Offset att_len;
 
    int att_ncid;
 
    int att_varid;
} hdf5_att_desc_t;
#endif
 
typedef struct file_desc_t
{
    iosystem_desc_t *iosystem;
 
    int fh;
 
#ifdef _ADIOS2
    char *filename;
 
    adios2_engine *engineH;
 
    int begin_step_called;
    int num_step_calls;;
    int max_step_calls;
 
    /*
     * Used to call adios2_end_step to avoid buffer overflow in MPI_Gatherv
     * during ADIOS metadata write operation.
     *
     * if num_written_blocks * BLOCK_METADATA_SIZE >= BLOCK_COUNT_THRESHOLD, call adios2_end_step
     * (Not implemented in this version. adios2_end_step is called if num_step_calls >= max_step_calls (= PIO_MAX_CACHED_STEPS_FOR_ADIOS))
     */
    unsigned int num_written_blocks;
 
    int write_decomp_id;
    int write_frame_id;
    int write_fillval_id;
 
    adios2_io *ioH;
 
    char transport[PIO_MAX_NAME];
 
    char params[PIO_MAX_NAME];
 
    char *dim_names[PIO_MAX_DIMS];
    PIO_Offset dim_values[PIO_MAX_DIMS];
 
    int num_dim_vars;
 
    struct adios_var_desc_t adios_vars[PIO_MAX_VARS];
 
    int num_vars;
 
    int num_gattrs;
 
    /* all procs rank, etc */
    MPI_Comm all_comm;
    int all_rank, num_alltasks;
 
    /* ADIOS rank, etc */
    int adios_io_process;
    MPI_Comm adios_comm;
    int adios_rank, num_adiostasks;
 
    /* Merging distributed array blocks to reduce I/O overhead */
    /* Grouping of processes for block merging */
    MPI_Comm block_comm;
    int block_myrank, block_nprocs;
    int *block_list;
 
    /* Buffers for merging distributed array blocks */
    unsigned int *array_counts;
    unsigned int array_counts_size;
    unsigned int *array_disp;
    unsigned int array_disp_size;
    char *block_array;
    size_t block_array_size;
 
    /* Track attributes */
    struct adios_att_desc_t adios_attrs[PIO_MAX_ATTRS];
    int num_attrs;
 
    int fillmode;
 
    /* Handle PIO_Offset */
    int pio_offset_size;
    adios2_type pio_offset_type;
 
    int n_written_ioids;
    int written_ioids[PIO_MAX_ADIOS_DECOMPS]; /* written_ioids[N] = ioid if that decomp has been already written, */
 
    int current_frame;
 
    /* Some caches (hash tables) for ADIOS read */
    struct spio_hmap *cache_data_blocks;
    struct spio_hmap *cache_block_sizes;
    struct spio_hmap *cache_darray_info;
 
    char io_name_reader[PIO_MAX_NAME + 1]; /* Name of io object, for ADIOS read */
    size_t adios_reader_num_decomp_blocks; /* Number of decomposition blocks, for ADIOS read */
 
    /* Indicates whether the decomposition maps (for ADIOS write) need to be stored in BP files. Default is true. */
    bool store_adios_decomp;
#endif /* _ADIOS2 */
 
#ifdef _HDF5
    hid_t hdf5_file_id;
 
    /* Collective dataset transfer property list, used by H5Dwrite */
    hid_t dxplid_coll;
 
    /* Independent dataset transfer property list, used by H5Dwrite */
    hid_t dxplid_indep;
 
    struct hdf5_dim_desc_t hdf5_dims[PIO_MAX_DIMS];
 
    int hdf5_num_dims;
 
    struct hdf5_var_desc_t hdf5_vars[PIO_MAX_VARS];
 
    int hdf5_num_vars;
 
    struct hdf5_att_desc_t hdf5_attrs[PIO_MAX_ATTRS];
 
    int hdf5_num_attrs;
 
    int hdf5_num_gattrs;
#endif /* _HDF5 */
 
    /* File name - cached */
    char fname[PIO_MAX_NAME + 1];
 
    int pio_ncid;
 
    int iotype;
 
    struct var_desc_t varlist[PIO_MAX_VARS];
 
    /* Number of unlimited dim ids, if no unlimited id present = 0 */
    int num_unlim_dimids;
 
    /* Unlimited dim ids, if no unlimited id present = NULL */
    int *unlim_dimids;
 
    int mode;
 
    struct wmulti_buffer buffer;
 
    /* Bytes pending to be read on this file*/
    PIO_Offset rb_pend;
 
    /* Bytes pending to be written out for this file */
    PIO_Offset wb_pend;
 
    void *mvcache;
 
    struct spio_io_fstats_summary *io_fstats;
 
    struct file_desc_t *next;
 
    int do_io;
 
    bool reserve_extra_header_space;
 
    bool is_reopened;
} file_desc_t;
 
enum PIO_IOTYPE
{
    PIO_IOTYPE_PNETCDF = 1,
 
    PIO_IOTYPE_NETCDF = 2,
 
    PIO_IOTYPE_NETCDF4C = 3,
 
    PIO_IOTYPE_NETCDF4P = 4,
 
    PIO_IOTYPE_NETCDF4P_NCZARR = 5,
 
    PIO_IOTYPE_ADIOS = 6,
 
    PIO_IOTYPE_ADIOSC = 7,
 
    PIO_IOTYPE_HDF5 = 8
};
 
enum PIO_REARRANGERS
{
    PIO_REARR_BOX = 1,
 
    PIO_REARR_SUBSET = 2,
 
    PIO_REARR_ANY = 3
};
 
enum PIO_ERROR_HANDLERS
{
    PIO_INTERNAL_ERROR = (-51),
 
    PIO_BCAST_ERROR = (-52),
 
    PIO_REDUCE_ERROR = (-53),
 
    PIO_RETURN_ERROR = (-54)
};
 
#include "pio_api.h"
 
#endif  // _PIO_H_