[mpich-commits] [mpich] MPICH primary repository branch, master, updated. v3.1b1-126-ge53c0f3

mysql vizuser noreply at mpich.org
Sat Oct 26 23:41:36 CDT 2013


This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "MPICH primary repository".

The branch, master has been updated
       via  e53c0f3a2485069ad29f97be0ae8b4e68bbf71b4 (commit)
       via  e858a6cbbe118079179491eb45d3296d4c40a537 (commit)
       via  e072e5ca3d2cca0a9028958b8f096bd436681d19 (commit)
       via  fb570792e9bc6705bb7ede2286402544544d3561 (commit)
       via  de70f872aa9b3c64c67aee1beeb5eadef2830b60 (commit)
       via  9fcd57127c9dfa50e7dd06162dbed9f131139de3 (commit)
       via  d5ca1acdb06e69de322fb87e2c13094ef35c2cc9 (commit)
       via  83f6c94a86abb4b8875e6ecc88e24d8a7ac0e8e4 (commit)
       via  601ac99d425e8d0dff6f584aac41e4878d2b405a (commit)
       via  fc51e3ad893e6ab11feec5b4f6edfd85d1773f02 (commit)
       via  255d0fb7f68e209def23ab90a3764b62f507e812 (commit)
       via  127ee1d283c738847e35120d462686b89845521e (commit)
       via  4c5f24c85b7bb51d53b9ddc2b8332cecfb948fe2 (commit)
       via  e39e4d2e87b47f1497e39fd626315fa3c4f2012f (commit)
       via  068ac9cd59b6658db9946527ec9d978ebf847279 (commit)
       via  b351a97e86367f2c21b59ee1a55e835a820d1374 (commit)
       via  15357039dbebe52910cf3afdc251f8deaadcf38d (commit)
       via  bfca138203fae7f64b3b1aeb9e2e1a0945a1b5aa (commit)
       via  21d84da991d484f7924935591416e77845331662 (commit)
       via  72bf15fbd0578876ea731ce8f5630b19d7af3132 (commit)
       via  39e5fa1537c97be827a33fe47d9935242b3cf849 (commit)
      from  7ff6a970336dea004f567410e2e81d22de3de10f (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.

- Log -----------------------------------------------------------------
http://git.mpich.org/mpich.git/commitdiff/e53c0f3a2485069ad29f97be0ae8b4e68bbf71b4

commit e53c0f3a2485069ad29f97be0ae8b4e68bbf71b4
Author: Pavan Balaji <balaji at mcs.anl.gov>
Date:   Sat Oct 26 21:07:19 2013 -0500

    Bug fix for the mpit_vars test.
    
    Some content was being printed even when not in verbose mode causing
    the test to incorrectly fail.
    
    Signed-off-by: Junchao Zhang <jczhang at mcs.anl.gov>

diff --git a/test/mpi/mpi_t/mpit_vars.c b/test/mpi/mpi_t/mpit_vars.c
index 6954466..3efd84e 100644
--- a/test/mpi/mpi_t/mpit_vars.c
+++ b/test/mpi/mpi_t/mpit_vars.c
@@ -187,10 +187,11 @@ int PrintCategories(FILE * fp)
         descLen = sizeof(desc);
         MPI_T_category_get_info(i, name, &nameLen, desc, &descLen, &numCvars,
                                 &numPvars, &numSubcat);
-        if (verbose)
+        if (verbose) {
             fprintf(fp, "Category %s has %d control variables, %d performance variables, %d subcategories\n",
                     name, numCvars, numPvars, numSubcat);
             fprintf(fp, "\tDescription: %s\n", desc);
+        }
 
         if (numCvars > 0) {
             if (verbose)

http://git.mpich.org/mpich.git/commitdiff/e858a6cbbe118079179491eb45d3296d4c40a537

commit e858a6cbbe118079179491eb45d3296d4c40a537
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Fri Oct 25 12:28:54 2013 -0500

    Rename mpich_params.{c,h} to mpich_cvars.{c,h}
    
    Also change src/util/param/params.yml to src/util/cvar/cvars.yml,
    maint/genparams to maint/gencvars
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/.gitignore b/.gitignore
index 9a0a2f0..681b08b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -224,8 +224,8 @@ Makefile.am-stamp
 /src/binding/f90/Makefile.mk
 
 # MPICH2 parameter handling
-/src/include/mpich_params.h
-/src/util/param/mpich_params.c
+/src/include/mpich_cvars.h
+/src/util/cvar/mpich_cvars.c
 
 
 ################################################################################
diff --git a/autogen.sh b/autogen.sh
index 2197ce7..fc36562 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -842,8 +842,8 @@ echo "done"
 
 # new parameter code
 echo_n "Generating parameter handling code... "
-if test -x maint/genparams -a "$do_getparms" = "yes" ; then
-    if ./maint/genparams ; then
+if test -x maint/gencvars -a "$do_getparms" = "yes" ; then
+    if ./maint/gencvars ; then
         echo "done"
     else
         echo "failed"
diff --git a/maint/Makefile.mk b/maint/Makefile.mk
index 8891964..d44777d 100644
--- a/maint/Makefile.mk
+++ b/maint/Makefile.mk
@@ -25,7 +25,7 @@ dist_noinst_SCRIPTS +=                \
     maint/gcovmerge.in                \
     maint/getcoverage.in              \
     maint/gen_subcfg_m4               \
-    maint/genparams                   \
+    maint/gencvars                   \
     maint/genstates.in                \
     maint/getcoverage.in              \
     maint/local_perl/lib/YAML/Tiny.pm \
diff --git a/maint/genparams b/maint/gencvars
similarity index 96%
rename from maint/genparams
rename to maint/gencvars
index 109879d..fbd3ae5 100755
--- a/maint/genparams
+++ b/maint/gencvars
@@ -7,7 +7,7 @@
 # script TODO:
 # - generate initialization function to read vals from env vars
 # - deal with string escaping in generated C strings
-# - sort/collate paramters by name/category
+# - sort/collate cvars by name/category
 
 use strict;
 use warnings;
@@ -46,12 +46,12 @@ my $alt_ns = "MPICH";
 # deprecated prefix for backward compatibility
 my $dep_ns = "MPIR_PARAM";
 
-# parameter description file
-my $cvar_file = "src/util/param/params.yml";
+# cvar description file
+my $cvar_file = "src/util/cvar/cvars.yml";
 
 # output source files
-my $header_file = "src/include/mpich_params.h";
-my $c_file      = "src/util/param/mpich_params.c";
+my $header_file = "src/include/mpich_cvars.h";
+my $c_file      = "src/util/cvar/mpich_cvars.c";
 my $readme_file = "README.envvar";
 
 GetOptions(
@@ -169,11 +169,11 @@ EOT
 die "missing 'cvars', stopped" unless exists $cvars->{cvars};
 
 print CVAR_HDR <<EOT;
-/* initializes parameter values from the environment */
+/* initializes cvar values from the environment */
 int ${fn_ns}_init(void);
 int ${fn_ns}_finalize(void);
 
-/* extern declarations for each parameter
+/* extern declarations for each cvar
  * (definitions in $c_file) */
 EOT
 
@@ -448,7 +448,7 @@ print CVAR_HDR <<EOT;
 /* arbitrary, simplifies interaction with external interfaces like MPI_T_ */
 #define ${uc_ns}_MAX_STRLEN (4096)
 
-/* helper macros for safely getting the default value of a parameter */
+/* helper macros for safely getting the default value of a cvar */
 EOT
 
 print CVAR_HDR <<EOT;
@@ -470,7 +470,7 @@ close(CVAR_README);
 ########################################################################
 # helper subroutines
 
-# transform a parameter type to a C-language type
+# transform a cvar type to a C-language type
 sub type2ctype {
     my $type = shift;
     my %typemap = (
@@ -492,11 +492,11 @@ sub fmt_default {
     my $type = shift;
 
     if (defined($literalval)) {
-        die "Both \"default\" and \"defaultliteral\" fields were specified for parameter \"$name\", stopped" if defined($val);
+        die "Both \"default\" and \"defaultliteral\" fields were specified for cvar \"$name\", stopped" if defined($val);
         return qq($literalval);
     }
 
-    die "Exactly one of \"default\" or \"defaultliteral\" fields must be specified for parameter \"$name\", stopped" unless defined($val);
+    die "Exactly one of \"default\" or \"defaultliteral\" fields must be specified for cvar \"$name\", stopped" unless defined($val);
 
     if ($type eq "string") {
         $val =~ s/"/\\"/g;
diff --git a/src/include/Makefile.mk b/src/include/Makefile.mk
index fab495e..3ff7407 100644
--- a/src/include/Makefile.mk
+++ b/src/include/Makefile.mk
@@ -34,7 +34,7 @@ noinst_HEADERS +=                   \
     src/include/mpiallstates.h      \
     src/include/mpibase.h           \
     src/include/mpibsend.h          \
-    src/include/mpich_params.h  \
+    src/include/mpich_cvars.h  \
     src/include/mpichconfconst.h    \
     src/include/mpidbg.h            \
     src/include/mpierror.h          \
@@ -67,5 +67,5 @@ noinst_HEADERS +=                   \
     src/include/oputil.h            \
     src/include/mpiinfo.h
 
-src/include/mpich_params.h: src/util/param/params.yml
-	$(top_srcdir)/maint/genparams
+src/include/mpich_cvars.h: src/util/cvar/cvars.yml
+	$(top_srcdir)/maint/gencvars
diff --git a/src/include/mpiimpl.h b/src/include/mpiimpl.h
index 00efd2e..d06f492 100644
--- a/src/include/mpiimpl.h
+++ b/src/include/mpiimpl.h
@@ -3715,8 +3715,8 @@ int MPID_VCR_Get_lpid(MPID_VCR vcr, int * lpid_ptr);
    file (mpiimpl.h). */
 #include "mpidpost.h"
 
-/* tunable parameter values */
-#include "mpich_params.h"
+/* tunable cvar values */
+#include "mpich_cvars.h"
 
 /* Tags for point to point operations which implement collective and other
    internal operations */
diff --git a/src/mpid/common/sock/mpidu_sock.h b/src/mpid/common/sock/mpidu_sock.h
index f9a22a3..a9a9e95 100644
--- a/src/mpid/common/sock/mpidu_sock.h
+++ b/src/mpid/common/sock/mpidu_sock.h
@@ -24,7 +24,7 @@ CPLUSPLUS_BEGIN
 #include "mpibase.h"
 #include "mpiutil.h"
 #include "mpitypedefs.h"
-#include "mpich_params.h"
+#include "mpich_cvars.h"
 /* implementation specific header file */    
 #include "mpidu_socki.h"
 
diff --git a/src/util/Makefile.mk b/src/util/Makefile.mk
index 39a64f2..7ae439e 100644
--- a/src/util/Makefile.mk
+++ b/src/util/Makefile.mk
@@ -11,7 +11,7 @@ include $(top_srcdir)/src/util/logging/Makefile.mk
 include $(top_srcdir)/src/util/mem/Makefile.mk
 include $(top_srcdir)/src/util/msgs/Makefile.mk
 include $(top_srcdir)/src/util/other/Makefile.mk
-include $(top_srcdir)/src/util/param/Makefile.mk
+include $(top_srcdir)/src/util/cvar/Makefile.mk
 include $(top_srcdir)/src/util/procmap/Makefile.mk
 include $(top_srcdir)/src/util/thread/Makefile.mk
 include $(top_srcdir)/src/util/wrappers/Makefile.mk
diff --git a/src/util/param/Makefile.mk b/src/util/cvar/Makefile.mk
similarity index 55%
rename from src/util/param/Makefile.mk
rename to src/util/cvar/Makefile.mk
index 60c3208..bd27c56 100644
--- a/src/util/param/Makefile.mk
+++ b/src/util/cvar/Makefile.mk
@@ -6,12 +6,12 @@
 ##
 
 lib_lib at MPILIBNAME@_la_SOURCES +=   \
-    src/util/param/mpich_params.c
+    src/util/cvar/mpich_cvars.c
 
-dist_noinst_DATA += src/util/param/params.yml
+dist_noinst_DATA += src/util/cvar/cvars.yml
 
 if MAINTAINER_MODE
 # normally built by autogen.sh, but this rebuild rule is here
-$(top_srcdir)/src/util/param/mpich_params.c: $(top_srcdir)/src/util/param/params.yml $(top_srcdir)/maint/genparams
-	( cd $(top_srcdir) && ./maint/genparams )
+$(top_srcdir)/src/util/cvar/mpich_cvars.c: $(top_srcdir)/src/util/cvar/cvars.yml $(top_srcdir)/maint/gencvars
+	( cd $(top_srcdir) && ./maint/gencvars )
 endif MAINTAINER_MODE
diff --git a/src/util/param/params.yml b/src/util/cvar/cvars.yml
similarity index 99%
rename from src/util/param/params.yml
rename to src/util/cvar/cvars.yml
index ee5576a..b69b8b7 100644
--- a/src/util/param/params.yml
+++ b/src/util/cvar/cvars.yml
@@ -5,7 +5,7 @@
 # features like tagging or anchors.
 #
 # It is used to generate cvar-handling code for MPICH.
-# See the maint/genparams script for more information.
+# See the maint/gencvars script for more information.
 #
 # The only tricky bit is that long reflowed text (such as a description)
 # should use ">-" to both fold newlines and remove the trailing newline.

http://git.mpich.org/mpich.git/commitdiff/e072e5ca3d2cca0a9028958b8f096bd436681d19

commit e072e5ca3d2cca0a9028958b8f096bd436681d19
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Fri Oct 25 10:47:42 2013 -0500

    Rename parameters MPIR_PARAM_* to MPIR_CVAR_*
    
    To adapt to naming for control variables in MPI_T.
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/maint/genparams b/maint/genparams
index dad5841..109879d 100755
--- a/maint/genparams
+++ b/maint/genparams
@@ -34,14 +34,20 @@ $Text::Wrap::unexpand = 0; # disable hard tabs in output
 # set true to enable debug output
 my $debug = 0;
 
+# namespace prefix for function names
+my $fn_ns = "MPIR_T_cvar";
+
 # namespace prefix for variable and type names
-my $ns = "MPIR_Param";
+my $ns = "MPIR_CVAR";
 # an alternative namespace used for environment variables, unused if set
 # to ""
 my $alt_ns = "MPICH";
 
+# deprecated prefix for backward compatibility
+my $dep_ns = "MPIR_PARAM";
+
 # parameter description file
-my $param_file = "src/util/param/params.yml";
+my $cvar_file = "src/util/param/params.yml";
 
 # output source files
 my $header_file = "src/include/mpich_params.h";
@@ -53,7 +59,7 @@ GetOptions(
     "debug!"      => \$debug,
     "namespace=s" => \$ns,
     "alt-namespace=s" => \$alt_ns,
-    "param-file"  => \$param_file,
+    "param-file"  => \$cvar_file,
     "header=s"    => \$header_file,
     "c-file=s"    => \$c_file,
     "readme-file=s" => \$readme_file,
@@ -68,7 +74,7 @@ Supported options:
     --help            - this output
     --debug           - enable some debugging output
     --namespace=STR   - use STR as variable/type prefix in generated code
-    --param-file=FILE - use FILE as input describing parameters
+    --param-file=FILE - use FILE as input describing cvars
 
 EOT
     exit 1;
@@ -85,17 +91,17 @@ my $uc_ns = uc($ns);
 # merge them into a single consistent configuration object
 
 my $yaml = YAML::Tiny->new();
-my $params = ($yaml->read($param_file))->[0]; # [0] is for the first document
-print Dumper($params)."\n" if $debug;
-die "not a HASH, stopped" unless ref($params) eq "HASH";
+my $cvars = ($yaml->read($cvar_file))->[0]; # [0] is for the first document
+print Dumper($cvars)."\n" if $debug;
+die "not a HASH, stopped" unless ref($cvars) eq "HASH";
 
 ########################################################################
 # validate the config file
 
 # only simple checks for now, just make sure that all categories
-# referenced by parameters actually exist
-my %cat_hash = (map { ($_->{name} => 1) } @{$params->{categories}});
-foreach my $p (@{$params->{parameters}}) {
+# referenced by cvars actually exist
+my %cat_hash = (map { ($_->{name} => 1) } @{$cvars->{categories}});
+foreach my $p (@{$cvars->{cvars}}) {
     unless (exists $cat_hash{$p->{category}}) {
         warn "category '".$p->{category}."' referenced by '".$p->{name}."' was not found";
     }
@@ -103,14 +109,14 @@ foreach my $p (@{$params->{parameters}}) {
 
 ########################################################################
 # setup output files
-open(PARAM_HDR,    '>', $header_file);
-open(PARAM_C,      '>', $c_file);
-open(PARAM_README, '>', $readme_file);
+open(CVAR_HDR,    '>', $header_file);
+open(CVAR_C,      '>', $c_file);
+open(CVAR_README, '>', $readme_file);
 
 my $hdr_guard = header_to_incl_guard($header_file);
-my $param_file_md5 = md5sum($param_file);
+my $cvar_file_md5 = md5sum($cvar_file);
 
-print PARAM_HDR <<EOT;
+print CVAR_HDR <<EOT;
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
 /*
  *  (C) 2011 by Argonne National Laboratory.
@@ -119,7 +125,7 @@ print PARAM_HDR <<EOT;
 /* automatically generated
  *   by:   $0
  *   on:   $run_timestamp
- *   from: $param_file (md5sum $param_file_md5)
+ *   from: $cvar_file (md5sum $cvar_file_md5)
  *
  * DO NOT EDIT!!!
  */
@@ -129,7 +135,7 @@ print PARAM_HDR <<EOT;
 
 EOT
 
-print PARAM_C <<EOT;
+print CVAR_C <<EOT;
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
 /*
  *  (C) 2011 by Argonne National Laboratory.
@@ -138,7 +144,7 @@ print PARAM_C <<EOT;
 /* automatically generated
  *   by:   $0
  *   at:   $run_timestamp
- *   from: $param_file (md5sum $param_file_md5)
+ *   from: $cvar_file (md5sum $cvar_file_md5)
  *
  * DO NOT EDIT!!!
  */
@@ -147,7 +153,7 @@ print PARAM_C <<EOT;
 
 EOT
 
-print PARAM_README <<EOT;
+print CVAR_README <<EOT;
 (C) 2011 by Argonne National Laboratory.
     See COPYRIGHT in top-level directory.
 
@@ -159,27 +165,27 @@ users.
 EOT
 
 ########################################################################
-# now the actual parameters
-die "missing 'parameters', stopped" unless exists $params->{parameters};
+# now the actual cvars
+die "missing 'cvars', stopped" unless exists $cvars->{cvars};
 
-print PARAM_HDR <<EOT;
+print CVAR_HDR <<EOT;
 /* initializes parameter values from the environment */
-int ${ns}_init_params(void);
-int ${ns}_finalize(void);
+int ${fn_ns}_init(void);
+int ${fn_ns}_finalize(void);
 
 /* extern declarations for each parameter
  * (definitions in $c_file) */
 EOT
 
 # XXX DJG TODO collate and separate by category
-foreach my $p (@{$params->{parameters}}) {
-    printf PARAM_HDR "extern %s ${uc_ns}_%s;\n",
+foreach my $p (@{$cvars->{cvars}}) {
+    printf CVAR_HDR "extern %s ${uc_ns}_%s;\n",
         type2ctype($p->{type}), $p->{name};
 }
 
-print PARAM_C "/* actual storage for parameters */\n";
+print CVAR_C "/* actual storage for cvars */\n";
 
-foreach my $p (@{$params->{parameters}}) {
+foreach my $p (@{$cvars->{cvars}}) {
     my $default;
     if ($p->{type} eq "string") {
         # handle strings specially to avoid various const issues
@@ -190,25 +196,25 @@ foreach my $p (@{$params->{parameters}}) {
     }
 
     if ($p->{class} eq "device") {
-	printf PARAM_C "#if defined MPID_%s\n", $p->{name};
-	printf PARAM_C "%s ${uc_ns}_%s = MPID_%s;\n", type2ctype($p->{type}), $p->{name},
+	printf CVAR_C "#if defined MPID_%s\n", $p->{name};
+	printf CVAR_C "%s ${uc_ns}_%s = MPID_%s;\n", type2ctype($p->{type}), $p->{name},
 	               $p->{name};
-	printf PARAM_C "#else\n";
+	printf CVAR_C "#else\n";
     }
-    printf PARAM_C "%s ${uc_ns}_%s = %s;\n", type2ctype($p->{type}), $p->{name}, $default;
+    printf CVAR_C "%s ${uc_ns}_%s = %s;\n", type2ctype($p->{type}), $p->{name}, $default;
     if ($p->{class} eq "device") {
-	printf PARAM_C "#endif /* MPID_%s */\n\n", $p->{name};
+	printf CVAR_C "#endif /* MPID_%s */\n\n", $p->{name};
     }
 }
 
 # FIXME the mpi_errno bit is MPICH-specific
-print PARAM_C <<EOT;
+print CVAR_C <<EOT;
 
 #undef FUNCNAME
-#define FUNCNAME ${ns}_init_params
+#define FUNCNAME ${fn_ns}_init
 #undef FCNAME
 #define FCNAME MPIU_QUOTE(FUNCNAME)
-int ${ns}_init_params(void)
+int ${fn_ns}_init(void)
 {
     int mpi_errno = MPI_SUCCESS;
     int rc;
@@ -223,53 +229,53 @@ int ${ns}_init_params(void)
 
 EOT
 
-foreach my $p (@{$params->{parameters}}) {
+foreach my $p (@{$cvars->{cvars}}) {
     my $count = 1;
     my $mpi_dtype;
     if ($p->{type} eq 'string') {
         $mpi_dtype = "MPI_CHAR";
-        $count = "MPIR_PARAM_MAX_STRLEN";
+        $count = "MPIR_CVAR_MAX_STRLEN";
         my $str_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
-        printf PARAM_C qq(    defaultval.str = %s;\n), $str_val;
+        printf CVAR_C qq(    defaultval.str = %s;\n), $str_val;
     }
     elsif ($p->{type} eq 'int' or $p->{type} eq 'boolean') {
         $mpi_dtype = "MPI_INT";
         $count = 1;
         my $int_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
-        printf PARAM_C qq(    defaultval.d = %s;\n), $int_val;
+        printf CVAR_C qq(    defaultval.d = %s;\n), $int_val;
     }
     elsif ($p->{type} eq 'unsigned') {
         $mpi_dtype = "MPI_UNSINGED";
         $count = 1;
         my $double_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
-        printf PARAM_C qq(    defaultval.u = %s;\n), $double_val;
+        printf CVAR_C qq(    defaultval.u = %s;\n), $double_val;
     }
     elsif ($p->{type} eq 'unsigned long') {
         $mpi_dtype = "MPI_DOUBLE";
         $count = 1;
         my $double_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
-        printf PARAM_C qq(    defaultval.ul = %s;\n), $double_val;
+        printf CVAR_C qq(    defaultval.ul = %s;\n), $double_val;
     }
     elsif ($p->{type} eq 'unsigned long long') {
         $mpi_dtype = "MPI_DOUBLE";
         $count = 1;
         my $double_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
-        printf PARAM_C qq(    defaultval.ull = %s;\n), $double_val;
+        printf CVAR_C qq(    defaultval.ull = %s;\n), $double_val;
     }
     elsif ($p->{type} eq 'double') {
         $mpi_dtype = "MPI_DOUBLE";
         $count = 1;
         my $double_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
-        printf PARAM_C qq(    defaultval.d = %s;\n), $double_val;
+        printf CVAR_C qq(    defaultval.d = %s;\n), $double_val;
     }
     elsif ($p->{type} eq 'range') {
         $mpi_dtype = "MPI_INT";
         $count = 2;
         my $range_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
-        printf PARAM_C qq(    {\n);
-        printf PARAM_C qq(        MPIR_T_cvar_range_value_t tmp = %s;\n), $range_val;
-        printf PARAM_C qq(        defaultval.range = tmp;\n);
-        printf PARAM_C qq(    }\n);
+        printf CVAR_C qq(    {\n);
+        printf CVAR_C qq(        MPIR_T_cvar_range_value_t tmp = %s;\n), $range_val;
+        printf CVAR_C qq(        defaultval.range = tmp;\n);
+        printf CVAR_C qq(    }\n);
     }
     else {
         die "unknown type $p->{type}, stopped";
@@ -278,7 +284,7 @@ foreach my $p (@{$params->{parameters}}) {
     my $desc = $p->{description};
     $desc =~ s/"/\\"/g;
 
-    printf PARAM_C qq(    MPIR_T_CVAR_REGISTER_STATIC(\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s);\n),
+    printf CVAR_C qq(    MPIR_T_CVAR_REGISTER_STATIC(\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s);\n),
         qq(        $mpi_dtype,),
         qq(        ${uc_ns}_$p->{name}, /* name */),
         qq(        &${uc_ns}_$p->{name}, /* address */),
@@ -290,7 +296,7 @@ foreach my $p (@{$params->{parameters}}) {
         qq(        "$desc");
 
     if ($p->{type} eq 'string') {
-print PARAM_C <<EOT;
+print CVAR_C <<EOT;
     ${uc_ns}_GET_DEFAULT_STRING(${uc_ns}_$p->{name}, &tmp_str);
 EOT
     }
@@ -301,34 +307,36 @@ EOT
 
     # process extra envs first so the primary always wins
     push @env_names, @{$p->{'abs-alt-env'}} if $p->{'abs-alt-env'};
+    push @env_names, map { "${dep_ns}_$_" } @{$p->{'alt-env'}};
     push @env_names, map { "${alt_ns}_$_" } @{$p->{'alt-env'}};
     push @env_names, map { "${uc_ns}_$_" } @{$p->{'alt-env'}};
+    push @env_names, "${dep_ns}_" . $p->{name};
     push @env_names, "${alt_ns}_" . $p->{name};
     push @env_names, "${uc_ns}_" . $p->{name};
 
     foreach my $env_name (@env_names) {
         # assumes rc is defined
         if ($p->{type} eq 'range') {
-            print PARAM_C <<EOT;
+            print CVAR_C <<EOT;
     rc = MPL_env2${env_fn}("$env_name", &($var_name.low), &($var_name.high));
     MPIU_ERR_CHKANDJUMP1((-1 == rc),mpi_errno,MPI_ERR_OTHER,"**envvarparse","**envvarparse %s","$env_name");
 EOT
         }
         elsif ($p->{type} eq 'string') {
-            print PARAM_C <<EOT;
+            print CVAR_C <<EOT;
     rc = MPL_env2${env_fn}("$env_name", &tmp_str);
     MPIU_ERR_CHKANDJUMP1((-1 == rc),mpi_errno,MPI_ERR_OTHER,"**envvarparse","**envvarparse %s","$env_name");
 EOT
         }
         else {
-            print PARAM_C <<EOT;
+            print CVAR_C <<EOT;
     rc = MPL_env2${env_fn}("$env_name", &($var_name));
     MPIU_ERR_CHKANDJUMP1((-1 == rc),mpi_errno,MPI_ERR_OTHER,"**envvarparse","**envvarparse %s","$env_name");
 EOT
         }
     }
     if ($p->{type} eq 'string') {
-        print PARAM_C <<EOT;
+        print CVAR_C <<EOT;
     if (tmp_str != NULL) {
         ${var_name} = MPIU_Strdup(tmp_str);
         ${ns}_assert(${var_name});
@@ -342,18 +350,18 @@ EOT
     }
 EOT
     }
-    print PARAM_C "\n";
+    print CVAR_C "\n";
 }
 
-foreach my $cat (@{$params->{categories}}) {
+foreach my $cat (@{$cvars->{categories}}) {
     my $desc = $cat->{description};
     $desc =~ s/"/\\"/g;
-    printf PARAM_C qq(    MPIR_T_cat_add_desc(%s\n%s);\n\n),
+    printf CVAR_C qq(    MPIR_T_cat_add_desc(%s\n%s);\n\n),
         qq("$cat->{name}",),
         qq(        "$desc");
 }
 
-print PARAM_C <<EOT;
+print CVAR_C <<EOT;
 fn_exit:
     return mpi_errno;
 fn_fail:
@@ -362,19 +370,19 @@ fn_fail:
 
 EOT
 
-print PARAM_C <<EOT;
-int ${ns}_finalize(void)
+print CVAR_C <<EOT;
+int ${fn_ns}_finalize(void)
 {
     int mpi_errno = MPI_SUCCESS;
 
 EOT
 
-foreach my $p (@{$params->{parameters}}) {
+foreach my $p (@{$cvars->{cvars}}) {
     my $var_name = "${uc_ns}_" . $p->{name};
 
     if ($p->{type} eq "string") {
         # need to cleanup after whatever was strduped by the init routine
-print PARAM_C <<EOT;
+print CVAR_C <<EOT;
     if (${var_name} != NULL) {
         MPIU_Free(${var_name});
         ${var_name} = NULL;
@@ -385,7 +393,7 @@ EOT
 }
 
 
-print PARAM_C <<EOT;
+print CVAR_C <<EOT;
 fn_exit:
     return mpi_errno;
 fn_fail:
@@ -394,43 +402,45 @@ fn_fail:
 
 EOT
 
-foreach my $p (@{$params->{parameters}}) {
+foreach my $p (@{$cvars->{cvars}}) {
     my @env_names = ();
     my $first;
     my $alt;
     my $default;
 
     # process extra envs first so the primary always wins
+    push @env_names, "${dep_ns}_" . $p->{name};
     push @env_names, "${alt_ns}_" . $p->{name};
     push @env_names, map { "${uc_ns}_$_" } @{$p->{'alt-env'}};
     push @env_names, map { "${alt_ns}_$_" } @{$p->{'alt-env'}};
+    push @env_names, map { "${dep_ns}_$_" } @{$p->{'alt-env'}};
     push @env_names, @{$p->{'abs-alt-env'}} if $p->{'abs-alt-env'};
 
-    print PARAM_README "${uc_ns}_$p->{name}\n";
+    print CVAR_README "${uc_ns}_$p->{name}\n";
 
     $first = 1;
     foreach $alt (@env_names) {
         if ($first) {
-            print PARAM_README "    Aliases: $alt\n";
+            print CVAR_README "    Aliases: $alt\n";
         } else {
-            print PARAM_README "             $alt\n";
+            print CVAR_README "             $alt\n";
         }
         $first = 0;
     }
 
-    print PARAM_README wrap("    Description: ", "        ", $p->{description} . "\n");
+    print CVAR_README wrap("    Description: ", "        ", $p->{description} . "\n");
     $default = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
-    print PARAM_README "    Default: $default\n";
-    print PARAM_README "\n";
+    print CVAR_README "    Default: $default\n";
+    print CVAR_README "\n";
 }
 
 
 ########################################################################
 # clean up
 
-close(PARAM_C);
+close(CVAR_C);
 
-print PARAM_HDR <<EOT;
+print CVAR_HDR <<EOT;
 
 /* TODO: this should be defined elsewhere */
 #define ${ns}_assert MPIU_Assert
@@ -441,21 +451,21 @@ print PARAM_HDR <<EOT;
 /* helper macros for safely getting the default value of a parameter */
 EOT
 
-print PARAM_HDR <<EOT;
+print CVAR_HDR <<EOT;
 #endif /* $hdr_guard */
 EOT
-close(PARAM_HDR);
+close(CVAR_HDR);
 
-print PARAM_README <<EOT;
+print CVAR_README <<EOT;
 ---------------------------------------------------------------------------
 
 Automatically generated
   by:   $0
   at:   $run_timestamp
-  from: $param_file (md5sum $param_file_md5)
+  from: $cvar_file (md5sum $cvar_file_md5)
 
 EOT
-close(PARAM_README);
+close(CVAR_README);
 
 ########################################################################
 # helper subroutines
diff --git a/src/include/mpit.h b/src/include/mpit.h
index ea97e31..44d68ab 100644
--- a/src/include/mpit.h
+++ b/src/include/mpit.h
@@ -50,48 +50,48 @@ static inline cvar_table_entry_t * LOOKUP_CVAR_BY_NAME(const char* cvar_name)
 }
 
 /* Helper macros for getting the default value of a cvar */
-#define MPIR_PARAM_GET_DEFAULT_INT(name_,out_ptr_)   \
+#define MPIR_CVAR_GET_DEFAULT_INT(name_,out_ptr_)   \
     do {  \
         cvar_table_entry_t *cvar = LOOKUP_CVAR_BY_NAME(#name_); \
         *(out_ptr_) = cvar->defaultval.d; \
     } while (0)
 
-#define MPIR_PARAM_GET_DEFAULT_BOOLEAN(name_,out_ptr_)   \
+#define MPIR_CVAR_GET_DEFAULT_BOOLEAN(name_,out_ptr_)   \
     do {  \
-        MPIR_PARAM_GET_DEFAULT_INT(name_,out_ptr_); \
+        MPIR_CVAR_GET_DEFAULT_INT(name_,out_ptr_); \
     } while (0)
 
-#define MPIR_PARAM_GET_DEFAULT_UNSIGNED(name_,out_ptr_)   \
+#define MPIR_CVAR_GET_DEFAULT_UNSIGNED(name_,out_ptr_)   \
     do {  \
         cvar_table_entry_t *cvar = LOOKUP_CVAR_BY_NAME(#name_); \
         *(out_ptr_) = cvar->defaultval.u; \
     } while (0)
 
-#define MPIR_PARAM_GET_DEFAULT_UNSIGNED_LONG(name_,out_ptr_)   \
+#define MPIR_CVAR_GET_DEFAULT_UNSIGNED_LONG(name_,out_ptr_)   \
     do {  \
         cvar_table_entry_t *cvar = LOOKUP_CVAR_BY_NAME(#name_); \
         *(out_ptr_) = cvar->defaultval.ul; \
     } while (0)
 
-#define MPIR_PARAM_GET_DEFAULT_UNSIGNED_LONG_LONG(name_,out_ptr_)   \
+#define MPIR_CVAR_GET_DEFAULT_UNSIGNED_LONG_LONG(name_,out_ptr_)   \
     do {  \
         cvar_table_entry_t *cvar = LOOKUP_CVAR_BY_NAME(#name_); \
         *(out_ptr_) = cvar->defaultval.ull; \
     } while (0)
 
-#define MPIR_PARAM_GET_DEFAULT_DOUBLE(name_,out_ptr_)   \
+#define MPIR_CVAR_GET_DEFAULT_DOUBLE(name_,out_ptr_)   \
     do {  \
         cvar_table_entry_t *cvar = LOOKUP_CVAR_BY_NAME(#name_); \
         *(out_ptr_) = cvar->defaultval.f; \
     } while (0)
 
-#define MPIR_PARAM_GET_DEFAULT_STRING(name_,out_ptr_)   \
+#define MPIR_CVAR_GET_DEFAULT_STRING(name_,out_ptr_)   \
     do {  \
         cvar_table_entry_t *cvar = LOOKUP_CVAR_BY_NAME(#name_); \
         *(out_ptr_) = cvar->defaultval.str; \
     } while (0)
 
-#define MPIR_PARAM_GET_DEFAULT_RANGE(name_,out_ptr_)   \
+#define MPIR_CVAR_GET_DEFAULT_RANGE(name_,out_ptr_)   \
     do {  \
         cvar_table_entry_t *cvar = LOOKUP_CVAR_BY_NAME(#name_); \
         *(out_ptr_) = cvar->defaultval.range; \
diff --git a/src/include/mpiutil.h b/src/include/mpiutil.h
index 6b6559a..9cd5d66 100644
--- a/src/include/mpiutil.h
+++ b/src/include/mpiutil.h
@@ -43,7 +43,7 @@ const char *MPIU_Strerror(int errnum);
  * MPIU_Busy_wait()
  *
  * Call this in every busy wait loop to periodically yield the processor.  The
- * MPIR_PARAM_NEMESIS_POLLS_BEFORE_YIELD parameter can be used to adjust the number of
+ * MPIR_CVAR_NEMESIS_POLLS_BEFORE_YIELD parameter can be used to adjust the number of
  * times MPIU_Busy_wait is called before the yield function is called.
  */
 #ifdef USE_NOTHING_FOR_YIELD
@@ -54,9 +54,9 @@ const char *MPIU_Strerror(int errnum);
    need to be changed for fine-grained multithreading.  A possible alternative
    is to make it a global thread-local variable. */
 #define MPIU_Busy_wait() do {                                   \
-        if (MPIR_PARAM_NEMESIS_POLLS_BEFORE_YIELD) {                    \
+        if (MPIR_CVAR_NEMESIS_POLLS_BEFORE_YIELD) {                    \
             static int poll_count_ = 0;                         \
-            if (poll_count_ >= MPIR_PARAM_NEMESIS_POLLS_BEFORE_YIELD) { \
+            if (poll_count_ >= MPIR_CVAR_NEMESIS_POLLS_BEFORE_YIELD) { \
                 poll_count_ = 0;                                \
                 MPIU_PW_Sched_yield();                          \
             } else {                                            \
diff --git a/src/mpi/coll/allgather.c b/src/mpi/coll/allgather.c
index 9a3f30f..a7fc8e3 100644
--- a/src/mpi/coll/allgather.c
+++ b/src/mpi/coll/allgather.c
@@ -121,7 +121,7 @@ int MPIR_Allgather_intra (
     MPIDU_ERR_CHECK_MULTIPLE_THREADS_ENTER( comm_ptr );
 
     tot_bytes = (MPI_Aint)recvcount * comm_size * type_size;
-    if ((tot_bytes < MPIR_PARAM_ALLGATHER_LONG_MSG_SIZE) && !(comm_size & (comm_size - 1))) {
+    if ((tot_bytes < MPIR_CVAR_ALLGATHER_LONG_MSG_SIZE) && !(comm_size & (comm_size - 1))) {
 
         /* Short or medium size message and power-of-two no. of processes. Use
          * recursive doubling algorithm */   
@@ -446,7 +446,7 @@ int MPIR_Allgather_intra (
 #endif /* MPID_HAS_HETERO */
     }
 
-    else if (tot_bytes < MPIR_PARAM_ALLGATHER_SHORT_MSG_SIZE) {
+    else if (tot_bytes < MPIR_CVAR_ALLGATHER_SHORT_MSG_SIZE) {
         /* Short message and non-power-of-two no. of processes. Use
          * Bruck algorithm (see description above). */
 
diff --git a/src/mpi/coll/allgatherv.c b/src/mpi/coll/allgatherv.c
index b09dddb..f2faa14 100644
--- a/src/mpi/coll/allgatherv.c
+++ b/src/mpi/coll/allgatherv.c
@@ -112,7 +112,7 @@ int MPIR_Allgatherv_intra (
     MPID_Datatype_get_extent_macro( recvtype, recvtype_extent );
     MPID_Datatype_get_size_macro(recvtype, recvtype_size);
 
-    if ((total_count*recvtype_size < MPIR_PARAM_ALLGATHER_LONG_MSG_SIZE) &&
+    if ((total_count*recvtype_size < MPIR_CVAR_ALLGATHER_LONG_MSG_SIZE) &&
         !(comm_size & (comm_size - 1))) {
         /* Short or medium size message and power-of-two no. of processes. Use
          * recursive doubling algorithm */   
@@ -510,7 +510,7 @@ int MPIR_Allgatherv_intra (
 
     }
 
-    else if (total_count*recvtype_size < MPIR_PARAM_ALLGATHER_SHORT_MSG_SIZE) {
+    else if (total_count*recvtype_size < MPIR_CVAR_ALLGATHER_SHORT_MSG_SIZE) {
         /* Short message and non-power-of-two no. of processes. Use
          * Bruck algorithm (see description above). */
  
@@ -644,8 +644,8 @@ int MPIR_Allgatherv_intra (
 	for (i = 1; i < comm_size; i++)
 	    if (min > recvcounts[i])
                 min = recvcounts[i];
-	if (min * recvtype_extent < MPIR_PARAM_ALLGATHERV_PIPELINE_MSG_SIZE)
-	    min = MPIR_PARAM_ALLGATHERV_PIPELINE_MSG_SIZE / recvtype_extent;
+	if (min * recvtype_extent < MPIR_CVAR_ALLGATHERV_PIPELINE_MSG_SIZE)
+	    min = MPIR_CVAR_ALLGATHERV_PIPELINE_MSG_SIZE / recvtype_extent;
         /* Handle the case where the datatype extent is larger than
          * the pipeline size. */
         if (!min)
diff --git a/src/mpi/coll/allred_group.c b/src/mpi/coll/allred_group.c
index e05d36e..a493bc1 100644
--- a/src/mpi/coll/allred_group.c
+++ b/src/mpi/coll/allred_group.c
@@ -141,7 +141,7 @@ int MPIR_Allreduce_group_intra(void *sendbuf, void *recvbuf, int count,
        using recursive doubling in that case.) */
 
     if (newrank != -1) {
-        if ((count*type_size <= MPIR_PARAM_ALLREDUCE_SHORT_MSG_SIZE) ||
+        if ((count*type_size <= MPIR_CVAR_ALLREDUCE_SHORT_MSG_SIZE) ||
             (HANDLE_GET_KIND(op) != HANDLE_KIND_BUILTIN) ||
             (count < pof2))
         {
diff --git a/src/mpi/coll/allreduce.c b/src/mpi/coll/allreduce.c
index 377b692..766caa8 100644
--- a/src/mpi/coll/allreduce.c
+++ b/src/mpi/coll/allreduce.c
@@ -155,12 +155,12 @@ int MPIR_Allreduce_intra (
 
     is_commutative = MPIR_Op_is_commutative(op);
 
-    if (MPIR_PARAM_ENABLE_SMP_COLLECTIVES && MPIR_PARAM_ENABLE_SMP_ALLREDUCE) {
+    if (MPIR_CVAR_ENABLE_SMP_COLLECTIVES && MPIR_CVAR_ENABLE_SMP_ALLREDUCE) {
     /* is the op commutative? We do SMP optimizations only if it is. */
     MPID_Datatype_get_size_macro(datatype, type_size);
-    nbytes = MPIR_PARAM_MAX_SMP_ALLREDUCE_MSG_SIZE ? type_size*count : 0;
+    nbytes = MPIR_CVAR_MAX_SMP_ALLREDUCE_MSG_SIZE ? type_size*count : 0;
     if (MPIR_Comm_is_node_aware(comm_ptr) && is_commutative &&
-        nbytes <= MPIR_PARAM_MAX_SMP_ALLREDUCE_MSG_SIZE) {
+        nbytes <= MPIR_CVAR_MAX_SMP_ALLREDUCE_MSG_SIZE) {
         /* on each node, do a reduce to the local root */ 
         if (comm_ptr->node_comm != NULL) {
             /* take care of the MPI_IN_PLACE case. For reduce, 
@@ -343,7 +343,7 @@ int MPIR_Allreduce_intra (
            using recursive doubling in that case.) */
 
         if (newrank != -1) {
-            if ((count*type_size <= MPIR_PARAM_ALLREDUCE_SHORT_MSG_SIZE) ||
+            if ((count*type_size <= MPIR_CVAR_ALLREDUCE_SHORT_MSG_SIZE) ||
                 (HANDLE_GET_KIND(op) != HANDLE_KIND_BUILTIN) ||  
                 (count < pof2)) { /* use recursive doubling */
                 mask = 0x1;
diff --git a/src/mpi/coll/alltoall.c b/src/mpi/coll/alltoall.c
index 8d3f2f1..921852b 100644
--- a/src/mpi/coll/alltoall.c
+++ b/src/mpi/coll/alltoall.c
@@ -164,7 +164,7 @@ int MPIR_Alltoall_intra(
             }
         }
     }
-    else if ((nbytes <= MPIR_PARAM_ALLTOALL_SHORT_MSG_SIZE) && (comm_size >= 8)) {
+    else if ((nbytes <= MPIR_CVAR_ALLTOALL_SHORT_MSG_SIZE) && (comm_size >= 8)) {
 
         /* use the indexing algorithm by Jehoshua Bruck et al,
          * IEEE TPDS, Nov. 97 */ 
@@ -426,7 +426,7 @@ int MPIR_Alltoall_intra(
 
     }
 
-    else if (nbytes <= MPIR_PARAM_ALLTOALL_MEDIUM_MSG_SIZE) {
+    else if (nbytes <= MPIR_CVAR_ALLTOALL_MEDIUM_MSG_SIZE) {
         /* Medium-size message. Use isend/irecv with scattered
            destinations. Use Tony Ladd's modification to post only
            a small number of isends/irecvs at a time. */
@@ -442,7 +442,7 @@ int MPIR_Alltoall_intra(
 	 */
         int ii, ss, bblock;
 
-        bblock = MPIR_PARAM_ALLTOALL_THROTTLE;
+        bblock = MPIR_CVAR_ALLTOALL_THROTTLE;
         if (bblock == 0) bblock = comm_size;
 
         MPIU_CHKLMEM_MALLOC(reqarray, MPI_Request *, 2*bblock*sizeof(MPI_Request), mpi_errno, "reqarray");
diff --git a/src/mpi/coll/alltoallv.c b/src/mpi/coll/alltoallv.c
index a6ab621..359cd41 100644
--- a/src/mpi/coll/alltoallv.c
+++ b/src/mpi/coll/alltoallv.c
@@ -130,7 +130,7 @@ int MPIR_Alltoallv_intra(const void *sendbuf, const int *sendcounts, const int *
         }
     }
     else {
-        bblock = MPIR_PARAM_ALLTOALL_THROTTLE;
+        bblock = MPIR_CVAR_ALLTOALL_THROTTLE;
         if (bblock == 0) bblock = comm_size;
 
         MPID_Datatype_get_extent_macro(sendtype, send_extent);
diff --git a/src/mpi/coll/alltoallw.c b/src/mpi/coll/alltoallw.c
index 89ac201..f0e4565 100644
--- a/src/mpi/coll/alltoallw.c
+++ b/src/mpi/coll/alltoallw.c
@@ -120,7 +120,7 @@ int MPIR_Alltoallw_intra(const void *sendbuf, const int sendcounts[], const int
         }
     }
     else {
-        bblock = MPIR_PARAM_ALLTOALL_THROTTLE;
+        bblock = MPIR_CVAR_ALLTOALL_THROTTLE;
         if (bblock == 0) bblock = comm_size;
 
         MPIU_CHKLMEM_MALLOC(starray,  MPI_Status*,  2*bblock*sizeof(MPI_Status),  mpi_errno, "starray");
diff --git a/src/mpi/coll/barrier.c b/src/mpi/coll/barrier.c
index aec6cb0..efcc4fd 100644
--- a/src/mpi/coll/barrier.c
+++ b/src/mpi/coll/barrier.c
@@ -270,7 +270,7 @@ int MPIR_Barrier_impl(MPID_Comm *comm_ptr, int *errflag)
     else
     {
         if (comm_ptr->comm_kind == MPID_INTRACOMM) {
-            if (MPIR_PARAM_ENABLE_SMP_COLLECTIVES && MPIR_PARAM_ENABLE_SMP_BARRIER &&
+            if (MPIR_CVAR_ENABLE_SMP_COLLECTIVES && MPIR_CVAR_ENABLE_SMP_BARRIER &&
                 MPIR_Comm_is_node_aware(comm_ptr)) {
 
                 /* do the intranode barrier on all nodes */
diff --git a/src/mpi/coll/bcast.c b/src/mpi/coll/bcast.c
index 0a97422..aa3ec5e 100644
--- a/src/mpi/coll/bcast.c
+++ b/src/mpi/coll/bcast.c
@@ -917,7 +917,7 @@ static int MPIR_SMP_Bcast(
     MPI_Status status;
     int recvd_size;
 
-    if (!MPIR_PARAM_ENABLE_SMP_COLLECTIVES || !MPIR_PARAM_ENABLE_SMP_BCAST)
+    if (!MPIR_CVAR_ENABLE_SMP_COLLECTIVES || !MPIR_CVAR_ENABLE_SMP_BCAST)
         MPIU_Assert(0);
     MPIU_Assert(MPIR_Comm_is_node_aware(comm_ptr));
 
@@ -945,7 +945,7 @@ static int MPIR_SMP_Bcast(
     if (nbytes == 0)
         goto fn_exit; /* nothing to do */
 
-    if ((nbytes < MPIR_PARAM_BCAST_SHORT_MSG_SIZE) || (comm_ptr->local_size < MPIR_PARAM_BCAST_MIN_PROCS))
+    if ((nbytes < MPIR_CVAR_BCAST_SHORT_MSG_SIZE) || (comm_ptr->local_size < MPIR_CVAR_BCAST_MIN_PROCS))
     {
         /* send to intranode-rank 0 on the root's node */
         if (comm_ptr->node_comm != NULL &&
@@ -1001,12 +1001,12 @@ static int MPIR_SMP_Bcast(
                                       buffer, count, datatype, 0, comm_ptr->node_comm, errflag);
         }
     }
-    else /* (nbytes > MPIR_PARAM_BCAST_SHORT_MSG_SIZE) && (comm_ptr->size >= MPIR_PARAM_BCAST_MIN_PROCS) */
+    else /* (nbytes > MPIR_CVAR_BCAST_SHORT_MSG_SIZE) && (comm_ptr->size >= MPIR_CVAR_BCAST_MIN_PROCS) */
     {
         /* supposedly...
            smp+doubling good for pof2
            reg+ring better for non-pof2 */
-        if (nbytes < MPIR_PARAM_BCAST_LONG_MSG_SIZE && MPIU_is_pof2(comm_ptr->local_size, NULL))
+        if (nbytes < MPIR_CVAR_BCAST_LONG_MSG_SIZE && MPIU_is_pof2(comm_ptr->local_size, NULL))
         {
             /* medium-sized msg and pof2 np */
 
@@ -1151,9 +1151,9 @@ int MPIR_Bcast_intra (
     if (count == 0) goto fn_exit;
 
     MPID_Datatype_get_size_macro(datatype, type_size);
-    nbytes = MPIR_PARAM_MAX_SMP_BCAST_MSG_SIZE ? type_size*count : 0;
-    if (MPIR_PARAM_ENABLE_SMP_COLLECTIVES && MPIR_PARAM_ENABLE_SMP_BCAST &&
-        nbytes <= MPIR_PARAM_MAX_SMP_BCAST_MSG_SIZE && MPIR_Comm_is_node_aware(comm_ptr)) {
+    nbytes = MPIR_CVAR_MAX_SMP_BCAST_MSG_SIZE ? type_size*count : 0;
+    if (MPIR_CVAR_ENABLE_SMP_COLLECTIVES && MPIR_CVAR_ENABLE_SMP_BCAST &&
+        nbytes <= MPIR_CVAR_MAX_SMP_BCAST_MSG_SIZE && MPIR_Comm_is_node_aware(comm_ptr)) {
         mpi_errno = MPIR_SMP_Bcast(buffer, count, datatype, root, comm_ptr, errflag);
         if (mpi_errno) {
             /* for communication errors, just record the error but continue */
@@ -1190,7 +1190,7 @@ int MPIR_Bcast_intra (
     if (nbytes == 0)
         goto fn_exit; /* nothing to do */
 
-    if ((nbytes < MPIR_PARAM_BCAST_SHORT_MSG_SIZE) || (comm_size < MPIR_PARAM_BCAST_MIN_PROCS))
+    if ((nbytes < MPIR_CVAR_BCAST_SHORT_MSG_SIZE) || (comm_size < MPIR_CVAR_BCAST_MIN_PROCS))
     {
         mpi_errno = MPIR_Bcast_binomial(buffer, count, datatype, root, comm_ptr, errflag);
         if (mpi_errno) {
@@ -1200,9 +1200,9 @@ int MPIR_Bcast_intra (
             MPIU_ERR_ADD(mpi_errno_ret, mpi_errno);
         }
     }
-    else /* (nbytes >= MPIR_PARAM_BCAST_SHORT_MSG_SIZE) && (comm_size >= MPIR_PARAM_BCAST_MIN_PROCS) */
+    else /* (nbytes >= MPIR_CVAR_BCAST_SHORT_MSG_SIZE) && (comm_size >= MPIR_CVAR_BCAST_MIN_PROCS) */
     {
-        if ((nbytes < MPIR_PARAM_BCAST_LONG_MSG_SIZE) && (MPIU_is_pof2(comm_size, NULL)))
+        if ((nbytes < MPIR_CVAR_BCAST_LONG_MSG_SIZE) && (MPIU_is_pof2(comm_size, NULL)))
         {
             mpi_errno = MPIR_Bcast_scatter_doubling_allgather(buffer, count, datatype, root, comm_ptr, errflag);
             if (mpi_errno) {
@@ -1212,7 +1212,7 @@ int MPIR_Bcast_intra (
                 MPIU_ERR_ADD(mpi_errno_ret, mpi_errno);
             }
         }
-        else /* (nbytes >= MPIR_PARAM_BCAST_LONG_MSG_SIZE) || !(comm_size_is_pof2) */
+        else /* (nbytes >= MPIR_CVAR_BCAST_LONG_MSG_SIZE) || !(comm_size_is_pof2) */
         {
             /* We want the ring algorithm whether or not we have a
                topologically aware communicator.  Doing inter/intra-node
diff --git a/src/mpi/coll/gather.c b/src/mpi/coll/gather.c
index d8ca2ec..f7f9bbc 100644
--- a/src/mpi/coll/gather.c
+++ b/src/mpi/coll/gather.c
@@ -131,12 +131,12 @@ int MPIR_Gather_intra(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
 
 	/* If the message is smaller than the threshold, we will copy
 	 * our message in there too */
-	if (nbytes < MPIR_PARAM_GATHER_VSMALL_MSG_SIZE) tmp_buf_size++;
+	if (nbytes < MPIR_CVAR_GATHER_VSMALL_MSG_SIZE) tmp_buf_size++;
 
 	tmp_buf_size *= nbytes;
 
 	/* For zero-ranked root, we don't need any temporary buffer */
-	if ((rank == root) && (!root || (nbytes >= MPIR_PARAM_GATHER_VSMALL_MSG_SIZE)))
+	if ((rank == root) && (!root || (nbytes >= MPIR_CVAR_GATHER_VSMALL_MSG_SIZE)))
 	    tmp_buf_size = 0;
 
 	if (tmp_buf_size) {
@@ -152,7 +152,7 @@ int MPIR_Gather_intra(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
 		if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
 	    }
         }
-	else if (tmp_buf_size && (nbytes < MPIR_PARAM_GATHER_VSMALL_MSG_SIZE))
+	else if (tmp_buf_size && (nbytes < MPIR_CVAR_GATHER_VSMALL_MSG_SIZE))
 	{
             /* copy from sendbuf into tmp_buf */
             mpi_errno = MPIR_Localcopy(sendbuf, sendcount, sendtype,
@@ -196,7 +196,7 @@ int MPIR_Gather_intra(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                 MPIU_ERR_ADD(mpi_errno_ret, mpi_errno);
                             }
 			}
-			else if (nbytes < MPIR_PARAM_GATHER_VSMALL_MSG_SIZE) {
+			else if (nbytes < MPIR_CVAR_GATHER_VSMALL_MSG_SIZE) {
 			    mpi_errno = MPIC_Recv(tmp_buf, recvblks * nbytes, MPI_BYTE,
                                                      src, MPIR_GATHER_TAG, comm, &status, errflag);
                             if (mpi_errno) {
@@ -242,7 +242,7 @@ int MPIR_Gather_intra(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
 			if (relative_src + mask > comm_size)
 			    recvblks -= (relative_src + mask - comm_size);
 
-			if (nbytes < MPIR_PARAM_GATHER_VSMALL_MSG_SIZE)
+			if (nbytes < MPIR_CVAR_GATHER_VSMALL_MSG_SIZE)
 			    offset = mask * nbytes;
 			else
 			    offset = (mask - 1) * nbytes;
@@ -277,7 +277,7 @@ int MPIR_Gather_intra(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                         MPIU_ERR_ADD(mpi_errno_ret, mpi_errno);
                     }
                 }
-                else if (nbytes < MPIR_PARAM_GATHER_VSMALL_MSG_SIZE) {
+                else if (nbytes < MPIR_CVAR_GATHER_VSMALL_MSG_SIZE) {
 		    mpi_errno = MPIC_Send(tmp_buf, curr_cnt, MPI_BYTE, dst,
                                              MPIR_GATHER_TAG, comm, errflag);
                     if (mpi_errno) {
@@ -317,7 +317,7 @@ int MPIR_Gather_intra(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
             mask <<= 1;
         }
 
-        if ((rank == root) && root && (nbytes < MPIR_PARAM_GATHER_VSMALL_MSG_SIZE) && copy_blks)
+        if ((rank == root) && root && (nbytes < MPIR_CVAR_GATHER_VSMALL_MSG_SIZE) && copy_blks)
 	{
             /* reorder and copy from tmp_buf into recvbuf */
 	    mpi_errno = MPIR_Localcopy(tmp_buf,
@@ -504,7 +504,7 @@ int MPIR_Gather_inter(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
         nbytes = sendtype_size * sendcount * local_size;
     }
 
-    if (nbytes < MPIR_PARAM_GATHER_INTER_SHORT_MSG_SIZE)
+    if (nbytes < MPIR_CVAR_GATHER_INTER_SHORT_MSG_SIZE)
     {
         if (root == MPI_ROOT)
 	{
diff --git a/src/mpi/coll/gatherv.c b/src/mpi/coll/gatherv.c
index 15929d0..8801e03 100644
--- a/src/mpi/coll/gatherv.c
+++ b/src/mpi/coll/gatherv.c
@@ -141,11 +141,11 @@ int MPIR_Gatherv (
                irrelevant here. */
             comm_size = comm_ptr->local_size;
 
-            min_procs = MPIR_PARAM_GATHERV_INTER_SSEND_MIN_PROCS;
+            min_procs = MPIR_CVAR_GATHERV_INTER_SSEND_MIN_PROCS;
             if (min_procs == -1)
                 min_procs = comm_size + 1; /* Disable ssend */
             else if (min_procs == 0) /* backwards compatibility, use default value */
-                MPIR_PARAM_GET_DEFAULT_INT(GATHERV_INTER_SSEND_MIN_PROCS,&min_procs);
+                MPIR_CVAR_GET_DEFAULT_INT(GATHERV_INTER_SSEND_MIN_PROCS,&min_procs);
 
             if (comm_size >= min_procs) {
                 mpi_errno = MPIC_Ssend(sendbuf, sendcount, sendtype, root,
diff --git a/src/mpi/coll/helper_fns.c b/src/mpi/coll/helper_fns.c
index 89fbcf3..1c16730 100644
--- a/src/mpi/coll/helper_fns.c
+++ b/src/mpi/coll/helper_fns.c
@@ -267,7 +267,7 @@ int MPIC_Send(const void *buf, int count, MPI_Datatype datatype, int dest, int t
     MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
                          "**countneg", "**countneg %d", count);
 
-    if (*errflag && MPIR_PARAM_ENABLE_COLL_FT_RET)
+    if (*errflag && MPIR_CVAR_ENABLE_COLL_FT_RET)
         MPIR_TAG_SET_ERROR_BIT(tag);
 
     MPID_Comm_get_ptr(comm, comm_ptr);
@@ -335,7 +335,7 @@ int MPIC_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag,
         MPID_Request_release(request_ptr);
     }
 
-    if (!MPIR_PARAM_ENABLE_COLL_FT_RET) goto fn_exit;
+    if (!MPIR_CVAR_ENABLE_COLL_FT_RET) goto fn_exit;
 
     if (source != MPI_PROC_NULL) {
         if (MPIR_TAG_CHECK_ERROR_BIT(status->MPI_TAG)) {
@@ -381,7 +381,7 @@ int MPIC_Ssend(const void *buf, int count, MPI_Datatype datatype, int dest, int
     context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
         MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;
 
-    if (*errflag && MPIR_PARAM_ENABLE_COLL_FT_RET)
+    if (*errflag && MPIR_CVAR_ENABLE_COLL_FT_RET)
         MPIR_TAG_SET_ERROR_BIT(tag);
 
     mpi_errno = MPID_Ssend(buf, count, datatype, dest, tag, comm_ptr,
@@ -432,7 +432,7 @@ int MPIC_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
     context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
         MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;
 
-    if (MPIR_PARAM_ENABLE_COLL_FT_RET) {
+    if (MPIR_CVAR_ENABLE_COLL_FT_RET) {
         if (status == MPI_STATUS_IGNORE) status = &mystatus;
         if (*errflag) MPIR_TAG_SET_ERROR_BIT(sendtag);
     }
@@ -455,7 +455,7 @@ int MPIC_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
     MPID_Request_release(send_req_ptr);
     MPID_Request_release(recv_req_ptr);
 
-    if (!MPIR_PARAM_ENABLE_COLL_FT_RET) goto fn_exit;
+    if (!MPIR_CVAR_ENABLE_COLL_FT_RET) goto fn_exit;
 
     if (source != MPI_PROC_NULL) {
         if (MPIR_TAG_CHECK_ERROR_BIT(status->MPI_TAG)) {
@@ -510,7 +510,7 @@ int MPIC_Sendrecv_replace(void *buf, int count, MPI_Datatype datatype,
     MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
                          "**countneg", "**countneg %d", count);
 
-    if (MPIR_PARAM_ENABLE_COLL_FT_RET) {
+    if (MPIR_CVAR_ENABLE_COLL_FT_RET) {
         if (status == MPI_STATUS_IGNORE) status = &mystatus;
         if (*errflag) MPIR_TAG_SET_ERROR_BIT(sendtag);
     }
@@ -571,7 +571,7 @@ int MPIC_Sendrecv_replace(void *buf, int count, MPI_Datatype datatype,
     MPID_Request_release(sreq);
     MPID_Request_release(rreq);
 
-    if (!MPIR_PARAM_ENABLE_COLL_FT_RET) goto fn_exit;
+    if (!MPIR_CVAR_ENABLE_COLL_FT_RET) goto fn_exit;
     if (mpi_errno) MPIU_ERR_POP(mpi_errno);
     
     if (source != MPI_PROC_NULL) {
@@ -612,7 +612,7 @@ int MPIC_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int
     MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
                          "**countneg", "**countneg %d", count);
 
-    if (*errflag && MPIR_PARAM_ENABLE_COLL_FT_RET)
+    if (*errflag && MPIR_CVAR_ENABLE_COLL_FT_RET)
         MPIR_TAG_SET_ERROR_BIT(tag);
 
     MPID_Comm_get_ptr(comm, comm_ptr);
@@ -693,7 +693,7 @@ int MPIC_Waitall(int numreq, MPI_Request requests[], MPI_Status statuses[], int
     mpi_errno = MPIR_Waitall_impl(numreq, requests, statuses);
     if (mpi_errno) MPIU_ERR_POP(mpi_errno);
 
-    if (*errflag || !MPIR_PARAM_ENABLE_COLL_FT_RET)
+    if (*errflag || !MPIR_CVAR_ENABLE_COLL_FT_RET)
         goto fn_exit;
 
     for (i = 0; i < numreq; ++i) {
diff --git a/src/mpi/coll/iallgather.c b/src/mpi/coll/iallgather.c
index 8313e78..4e7673b 100644
--- a/src/mpi/coll/iallgather.c
+++ b/src/mpi/coll/iallgather.c
@@ -453,13 +453,13 @@ int MPIR_Iallgather_intra(const void *sendbuf, int sendcount, MPI_Datatype sendt
     MPID_Datatype_get_size_macro(recvtype, recvtype_size);
     tot_bytes = (MPI_Aint)recvcount * comm_size * recvtype_size;
 
-    if ((tot_bytes < MPIR_PARAM_ALLGATHER_LONG_MSG_SIZE) && !(comm_size & (comm_size - 1))) {
+    if ((tot_bytes < MPIR_CVAR_ALLGATHER_LONG_MSG_SIZE) && !(comm_size & (comm_size - 1))) {
         /* Short or medium size message and power-of-two no. of processes. Use
          * recursive doubling algorithm */
         mpi_errno = MPIR_Iallgather_rec_dbl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm_ptr, s);
         if (mpi_errno) MPIU_ERR_POP(mpi_errno);
     }
-    else if (tot_bytes < MPIR_PARAM_ALLGATHER_SHORT_MSG_SIZE) {
+    else if (tot_bytes < MPIR_CVAR_ALLGATHER_SHORT_MSG_SIZE) {
         /* Short message and non-power-of-two no. of processes. Use
          * Bruck algorithm (see description above). */
         mpi_errno = MPIR_Iallgather_bruck(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm_ptr, s);
diff --git a/src/mpi/coll/iallgatherv.c b/src/mpi/coll/iallgatherv.c
index 40f6f67..1f5a7af 100644
--- a/src/mpi/coll/iallgatherv.c
+++ b/src/mpi/coll/iallgatherv.c
@@ -453,8 +453,8 @@ int MPIR_Iallgatherv_ring(const void *sendbuf, int sendcount, MPI_Datatype sendt
     for (i = 1; i < comm_size; i++)
         if (min > recvcounts[i])
             min = recvcounts[i];
-    if (min * recvtype_extent < MPIR_PARAM_ALLGATHERV_PIPELINE_MSG_SIZE)
-        min = MPIR_PARAM_ALLGATHERV_PIPELINE_MSG_SIZE / recvtype_extent;
+    if (min * recvtype_extent < MPIR_CVAR_ALLGATHERV_PIPELINE_MSG_SIZE)
+        min = MPIR_CVAR_ALLGATHERV_PIPELINE_MSG_SIZE / recvtype_extent;
     /* Handle the case where the datatype extent is larger than
      * the pipeline size. */
     if (!min)
@@ -566,7 +566,7 @@ int MPIR_Iallgatherv_intra(const void *sendbuf, int sendcount, MPI_Datatype send
     if (total_count == 0)
         goto fn_exit;
 
-    if ((total_count*recvtype_size < MPIR_PARAM_ALLGATHER_LONG_MSG_SIZE) &&
+    if ((total_count*recvtype_size < MPIR_CVAR_ALLGATHER_LONG_MSG_SIZE) &&
         !(comm_size & (comm_size - 1)))
     {
         /* Short or medium size message and power-of-two no. of processes. Use
@@ -574,7 +574,7 @@ int MPIR_Iallgatherv_intra(const void *sendbuf, int sendcount, MPI_Datatype send
         mpi_errno = MPIR_Iallgatherv_rec_dbl(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm_ptr, s);
         if (mpi_errno) MPIU_ERR_POP(mpi_errno);
     }
-    else if (total_count*recvtype_size < MPIR_PARAM_ALLGATHER_SHORT_MSG_SIZE) {
+    else if (total_count*recvtype_size < MPIR_CVAR_ALLGATHER_SHORT_MSG_SIZE) {
         /* Short message and non-power-of-two no. of processes. Use
          * Bruck algorithm (see description above). */
         mpi_errno = MPIR_Iallgatherv_bruck(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm_ptr, s);
diff --git a/src/mpi/coll/iallreduce.c b/src/mpi/coll/iallreduce.c
index 29d573d..ffdd34a 100644
--- a/src/mpi/coll/iallreduce.c
+++ b/src/mpi/coll/iallreduce.c
@@ -467,7 +467,7 @@ int MPIR_Iallreduce_intra(const void *sendbuf, void *recvbuf, int count, MPI_Dat
        datatypes to do the reduce-scatter is tricky, therefore
        using recursive doubling in that case.) */
 
-    if ((count*type_size <= MPIR_PARAM_ALLREDUCE_SHORT_MSG_SIZE) ||
+    if ((count*type_size <= MPIR_CVAR_ALLREDUCE_SHORT_MSG_SIZE) ||
         (HANDLE_GET_KIND(op) != HANDLE_KIND_BUILTIN) ||
         (count < pof2))
     {
@@ -571,7 +571,7 @@ int MPIR_Iallreduce_SMP(const void *sendbuf, void *recvbuf, int count, MPI_Datat
     MPID_Comm *nc;
     MPID_Comm *nrc;
 
-    if (!MPIR_PARAM_ENABLE_SMP_COLLECTIVES || !MPIR_PARAM_ENABLE_SMP_ALLREDUCE)
+    if (!MPIR_CVAR_ENABLE_SMP_COLLECTIVES || !MPIR_CVAR_ENABLE_SMP_ALLREDUCE)
         MPID_Abort(comm_ptr, MPI_ERR_OTHER, 1, "SMP collectives are disabled!");
     MPIU_Assert(MPIR_Comm_is_node_aware(comm_ptr));
 
diff --git a/src/mpi/coll/ialltoall.c b/src/mpi/coll/ialltoall.c
index bf1ba73..efbecda 100644
--- a/src/mpi/coll/ialltoall.c
+++ b/src/mpi/coll/ialltoall.c
@@ -266,7 +266,7 @@ int MPIR_Ialltoall_perm_sr(const void *sendbuf, int sendcount, MPI_Datatype send
     MPID_Datatype_get_extent_macro(sendtype, sendtype_extent);
     MPID_Datatype_get_extent_macro(recvtype, recvtype_extent);
 
-    bblock = MPIR_PARAM_ALLTOALL_THROTTLE;
+    bblock = MPIR_CVAR_ALLTOALL_THROTTLE;
     if (bblock == 0) bblock = comm_size;
 
     for (ii = 0; ii < comm_size; ii += bblock) {
@@ -417,13 +417,13 @@ int MPIR_Ialltoall_intra(const void *sendbuf, int sendcount, MPI_Datatype sendty
         mpi_errno = MPIR_Ialltoall_inplace(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm_ptr, s);
         if (mpi_errno) MPIU_ERR_POP(mpi_errno);
     }
-    else if ((nbytes <= MPIR_PARAM_ALLTOALL_SHORT_MSG_SIZE) && (comm_size >= 8)) {
+    else if ((nbytes <= MPIR_CVAR_ALLTOALL_SHORT_MSG_SIZE) && (comm_size >= 8)) {
         /* use the indexing algorithm by Jehoshua Bruck et al,
          * IEEE TPDS, Nov. 97 */
         mpi_errno = MPIR_Ialltoall_bruck(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm_ptr, s);
         if (mpi_errno) MPIU_ERR_POP(mpi_errno);
     }
-    else if (nbytes <= MPIR_PARAM_ALLTOALL_MEDIUM_MSG_SIZE) {
+    else if (nbytes <= MPIR_CVAR_ALLTOALL_MEDIUM_MSG_SIZE) {
         /* Medium-size message. Use isend/irecv with scattered
            destinations. Use Tony Ladd's modification to post only
            a small number of isends/irecvs at a time. */
diff --git a/src/mpi/coll/ialltoallv.c b/src/mpi/coll/ialltoallv.c
index 0d72e01..ab06c43 100644
--- a/src/mpi/coll/ialltoallv.c
+++ b/src/mpi/coll/ialltoallv.c
@@ -104,7 +104,7 @@ int MPIR_Ialltoallv_intra(const void *sendbuf, const int sendcounts[], const int
         MPID_SCHED_BARRIER(s);
     }
     else {
-        bblock = MPIR_PARAM_ALLTOALL_THROTTLE;
+        bblock = MPIR_CVAR_ALLTOALL_THROTTLE;
         if (bblock == 0)
             bblock = comm_size;
 
diff --git a/src/mpi/coll/ialltoallw.c b/src/mpi/coll/ialltoallw.c
index 26a72b7..47a3c6b 100644
--- a/src/mpi/coll/ialltoallw.c
+++ b/src/mpi/coll/ialltoallw.c
@@ -124,7 +124,7 @@ int MPIR_Ialltoallw_intra(const void *sendbuf, const int sendcounts[], const int
         }
     }
     else {
-        bblock = MPIR_PARAM_ALLTOALL_THROTTLE;
+        bblock = MPIR_CVAR_ALLTOALL_THROTTLE;
         if (bblock == 0) bblock = comm_size;
 
         /* post only bblock isends/irecvs at a time as suggested by Tony Ladd */
diff --git a/src/mpi/coll/ibcast.c b/src/mpi/coll/ibcast.c
index e2b8263..f80d6d2 100644
--- a/src/mpi/coll/ibcast.c
+++ b/src/mpi/coll/ibcast.c
@@ -671,7 +671,7 @@ int MPIR_Ibcast_SMP(void *buffer, int count, MPI_Datatype datatype, int root, MP
     int mpi_errno = MPI_SUCCESS;
     int type_size, is_homogeneous;
 
-    if (!MPIR_PARAM_ENABLE_SMP_COLLECTIVES || !MPIR_PARAM_ENABLE_SMP_BCAST)
+    if (!MPIR_CVAR_ENABLE_SMP_COLLECTIVES || !MPIR_CVAR_ENABLE_SMP_BCAST)
         MPID_Abort(comm_ptr, MPI_ERR_OTHER, 1, "SMP collectives are disabled!");
     MPIU_Assert(MPIR_Comm_is_node_aware(comm_ptr));
 
@@ -770,15 +770,15 @@ int MPIR_Ibcast_intra(void *buffer, int count, MPI_Datatype datatype, int root,
     nbytes = type_size * count;
 
     /* simplistic implementation for now */
-    if ((nbytes < MPIR_PARAM_BCAST_SHORT_MSG_SIZE) ||
-        (comm_size < MPIR_PARAM_BCAST_MIN_PROCS))
+    if ((nbytes < MPIR_CVAR_BCAST_SHORT_MSG_SIZE) ||
+        (comm_size < MPIR_CVAR_BCAST_MIN_PROCS))
     {
         mpi_errno = MPIR_Ibcast_binomial(buffer, count, datatype, root, comm_ptr, s);
         if (mpi_errno) MPIU_ERR_POP(mpi_errno);
     }
-    else /* (nbytes >= MPIR_PARAM_BCAST_SHORT_MSG_SIZE) && (comm_size >= MPIR_PARAM_BCAST_MIN_PROCS) */
+    else /* (nbytes >= MPIR_CVAR_BCAST_SHORT_MSG_SIZE) && (comm_size >= MPIR_CVAR_BCAST_MIN_PROCS) */
     {
-        if ((nbytes < MPIR_PARAM_BCAST_LONG_MSG_SIZE) && (MPIU_is_pof2(comm_size, NULL))) {
+        if ((nbytes < MPIR_CVAR_BCAST_LONG_MSG_SIZE) && (MPIU_is_pof2(comm_size, NULL))) {
             mpi_errno = MPIR_Ibcast_scatter_rec_dbl_allgather(buffer, count, datatype, root, comm_ptr, s);
             if (mpi_errno) MPIU_ERR_POP(mpi_errno);
         }
diff --git a/src/mpi/coll/igather.c b/src/mpi/coll/igather.c
index ffdda09..e5f4512 100644
--- a/src/mpi/coll/igather.c
+++ b/src/mpi/coll/igather.c
@@ -115,12 +115,12 @@ int MPIR_Igather_binomial(const void *sendbuf, int sendcount, MPI_Datatype sendt
 
         /* If the message is smaller than the threshold, we will copy
          * our message in there too */
-        if (nbytes < MPIR_PARAM_GATHER_VSMALL_MSG_SIZE) tmp_buf_size++;
+        if (nbytes < MPIR_CVAR_GATHER_VSMALL_MSG_SIZE) tmp_buf_size++;
 
         tmp_buf_size *= nbytes;
 
         /* For zero-ranked root, we don't need any temporary buffer */
-        if ((rank == root) && (!root || (nbytes >= MPIR_PARAM_GATHER_VSMALL_MSG_SIZE)))
+        if ((rank == root) && (!root || (nbytes >= MPIR_CVAR_GATHER_VSMALL_MSG_SIZE)))
             tmp_buf_size = 0;
 
         if (tmp_buf_size) {
@@ -134,7 +134,7 @@ int MPIR_Igather_binomial(const void *sendbuf, int sendcount, MPI_Datatype sendt
                 if (mpi_errno) MPIU_ERR_POP(mpi_errno);
             }
         }
-        else if (tmp_buf_size && (nbytes < MPIR_PARAM_GATHER_VSMALL_MSG_SIZE)) {
+        else if (tmp_buf_size && (nbytes < MPIR_CVAR_GATHER_VSMALL_MSG_SIZE)) {
             /* copy from sendbuf into tmp_buf */
             mpi_errno = MPIR_Localcopy(sendbuf, sendcount, sendtype,
                                        tmp_buf, nbytes, MPI_BYTE);
@@ -167,7 +167,7 @@ int MPIR_Igather_binomial(const void *sendbuf, int sendcount, MPI_Datatype sendt
                             mpi_errno = MPID_Sched_barrier(s);
                             if (mpi_errno) MPIU_ERR_POP(mpi_errno);
                         }
-                        else if (nbytes < MPIR_PARAM_GATHER_VSMALL_MSG_SIZE) {
+                        else if (nbytes < MPIR_CVAR_GATHER_VSMALL_MSG_SIZE) {
                             mpi_errno = MPID_Sched_recv(tmp_buf, (recvblks * nbytes), MPI_BYTE, src, comm_ptr, s);
                             if (mpi_errno) MPIU_ERR_POP(mpi_errno);
                             mpi_errno = MPID_Sched_barrier(s);
@@ -204,7 +204,7 @@ int MPIR_Igather_binomial(const void *sendbuf, int sendcount, MPI_Datatype sendt
                         if (relative_src + mask > comm_size)
                             recvblks -= (relative_src + mask - comm_size);
 
-                        if (nbytes < MPIR_PARAM_GATHER_VSMALL_MSG_SIZE)
+                        if (nbytes < MPIR_CVAR_GATHER_VSMALL_MSG_SIZE)
                             offset = mask * nbytes;
                         else
                             offset = (mask - 1) * nbytes;
@@ -228,7 +228,7 @@ int MPIR_Igather_binomial(const void *sendbuf, int sendcount, MPI_Datatype sendt
                     mpi_errno = MPID_Sched_barrier(s);
                     if (mpi_errno) MPIU_ERR_POP(mpi_errno);
                 }
-                else if (nbytes < MPIR_PARAM_GATHER_VSMALL_MSG_SIZE) {
+                else if (nbytes < MPIR_CVAR_GATHER_VSMALL_MSG_SIZE) {
                     mpi_errno = MPID_Sched_send(tmp_buf, curr_cnt, MPI_BYTE, dst, comm_ptr, s);
                     if (mpi_errno) MPIU_ERR_POP(mpi_errno);
                     mpi_errno = MPID_Sched_barrier(s);
@@ -260,7 +260,7 @@ int MPIR_Igather_binomial(const void *sendbuf, int sendcount, MPI_Datatype sendt
             mask <<= 1;
         }
 
-        if ((rank == root) && root && (nbytes < MPIR_PARAM_GATHER_VSMALL_MSG_SIZE) && copy_blks) {
+        if ((rank == root) && root && (nbytes < MPIR_CVAR_GATHER_VSMALL_MSG_SIZE) && copy_blks) {
             /* reorder and copy from tmp_buf into recvbuf */
             /* FIXME why are there two copies here? */
             mpi_errno = MPID_Sched_copy(tmp_buf, nbytes * (comm_size - copy_offset), MPI_BYTE,
@@ -431,7 +431,7 @@ int MPIR_Igather_inter(const void *sendbuf, int sendcount, MPI_Datatype sendtype
         nbytes = sendtype_size * sendcount * local_size;
     }
 
-    if (nbytes < MPIR_PARAM_GATHER_INTER_SHORT_MSG_SIZE) {
+    if (nbytes < MPIR_CVAR_GATHER_INTER_SHORT_MSG_SIZE) {
         if (root == MPI_ROOT) {
             /* root receives data from rank 0 on remote group */
             mpi_errno = MPID_Sched_recv(recvbuf, recvcount*remote_size, recvtype, 0, comm_ptr, s);
diff --git a/src/mpi/coll/igatherv.c b/src/mpi/coll/igatherv.c
index f87d163..b7d5422 100644
--- a/src/mpi/coll/igatherv.c
+++ b/src/mpi/coll/igatherv.c
@@ -80,11 +80,11 @@ int MPIR_Igatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, voi
                irrelevant here. */
             comm_size = comm_ptr->local_size;
 
-            min_procs = MPIR_PARAM_GATHERV_INTER_SSEND_MIN_PROCS;
+            min_procs = MPIR_CVAR_GATHERV_INTER_SSEND_MIN_PROCS;
             if (min_procs == -1)
                 min_procs = comm_size + 1; /* Disable ssend */
             else if (min_procs == 0) /* backwards compatibility, use default value */
-                MPIR_PARAM_GET_DEFAULT_INT(GATHERV_INTER_SSEND_MIN_PROCS,&min_procs);
+                MPIR_CVAR_GET_DEFAULT_INT(GATHERV_INTER_SSEND_MIN_PROCS,&min_procs);
 
             if (comm_size >= min_procs)
                 mpi_errno = MPID_Sched_ssend(sendbuf, sendcount, sendtype, root, comm_ptr, s);
diff --git a/src/mpi/coll/ired_scat.c b/src/mpi/coll/ired_scat.c
index e430084..989cfa6 100644
--- a/src/mpi/coll/ired_scat.c
+++ b/src/mpi/coll/ired_scat.c
@@ -856,11 +856,11 @@ int MPIR_Ireduce_scatter_intra(const void *sendbuf, void *recvbuf, const int rec
     nbytes = total_count * type_size;
 
     /* select an appropriate algorithm based on commutivity and message size */
-    if (is_commutative && (nbytes < MPIR_PARAM_REDSCAT_COMMUTATIVE_LONG_MSG_SIZE)) {
+    if (is_commutative && (nbytes < MPIR_CVAR_REDSCAT_COMMUTATIVE_LONG_MSG_SIZE)) {
         mpi_errno = MPIR_Ireduce_scatter_rec_hlv(sendbuf, recvbuf, recvcounts, datatype, op, comm_ptr, s);
         if (mpi_errno) MPIU_ERR_POP(mpi_errno);
     }
-    else if (is_commutative && (nbytes >= MPIR_PARAM_REDSCAT_COMMUTATIVE_LONG_MSG_SIZE)) {
+    else if (is_commutative && (nbytes >= MPIR_CVAR_REDSCAT_COMMUTATIVE_LONG_MSG_SIZE)) {
         mpi_errno = MPIR_Ireduce_scatter_pairwise(sendbuf, recvbuf, recvcounts, datatype, op, comm_ptr, s);
         if (mpi_errno) MPIU_ERR_POP(mpi_errno);
     }
diff --git a/src/mpi/coll/ired_scat_block.c b/src/mpi/coll/ired_scat_block.c
index ac23912..3801ed1 100644
--- a/src/mpi/coll/ired_scat_block.c
+++ b/src/mpi/coll/ired_scat_block.c
@@ -781,11 +781,11 @@ int MPIR_Ireduce_scatter_block_intra(const void *sendbuf, void *recvbuf, int rec
     nbytes = total_count * type_size;
 
     /* select an appropriate algorithm based on commutivity and message size */
-    if (is_commutative && (nbytes < MPIR_PARAM_REDSCAT_COMMUTATIVE_LONG_MSG_SIZE)) {
+    if (is_commutative && (nbytes < MPIR_CVAR_REDSCAT_COMMUTATIVE_LONG_MSG_SIZE)) {
         mpi_errno = MPIR_Ireduce_scatter_block_rec_hlv(sendbuf, recvbuf, recvcount, datatype, op, comm_ptr, s);
         if (mpi_errno) MPIU_ERR_POP(mpi_errno);
     }
-    else if (is_commutative && (nbytes >= MPIR_PARAM_REDSCAT_COMMUTATIVE_LONG_MSG_SIZE)) {
+    else if (is_commutative && (nbytes >= MPIR_CVAR_REDSCAT_COMMUTATIVE_LONG_MSG_SIZE)) {
         mpi_errno = MPIR_Ireduce_scatter_block_pairwise(sendbuf, recvbuf, recvcount, datatype, op, comm_ptr, s);
         if (mpi_errno) MPIU_ERR_POP(mpi_errno);
     }
diff --git a/src/mpi/coll/ireduce.c b/src/mpi/coll/ireduce.c
index 0e05112..ff897e3 100644
--- a/src/mpi/coll/ireduce.c
+++ b/src/mpi/coll/ireduce.c
@@ -545,7 +545,7 @@ int MPIR_Ireduce_intra(const void *sendbuf, void *recvbuf, int count, MPI_Dataty
     while (pof2 <= comm_size) pof2 <<= 1;
     pof2 >>=1;
 
-    if ((count*type_size > MPIR_PARAM_REDUCE_SHORT_MSG_SIZE) &&
+    if ((count*type_size > MPIR_CVAR_REDUCE_SHORT_MSG_SIZE) &&
         (HANDLE_GET_KIND(op) == HANDLE_KIND_BUILTIN) &&
         (count >= pof2))
     {
@@ -579,7 +579,7 @@ int MPIR_Ireduce_SMP(const void *sendbuf, void *recvbuf, int count, MPI_Datatype
     MPID_Comm *nrc;
     MPIR_SCHED_CHKPMEM_DECL(1);
 
-    if (!MPIR_PARAM_ENABLE_SMP_COLLECTIVES || !MPIR_PARAM_ENABLE_SMP_REDUCE)
+    if (!MPIR_CVAR_ENABLE_SMP_COLLECTIVES || !MPIR_CVAR_ENABLE_SMP_REDUCE)
         MPID_Abort(comm_ptr, MPI_ERR_OTHER, 1, "SMP collectives are disabled!");
     MPIU_Assert(MPIR_Comm_is_node_aware(comm_ptr));
     MPIU_Assert(comm_ptr->comm_kind == MPID_INTRACOMM);
diff --git a/src/mpi/coll/iscatter.c b/src/mpi/coll/iscatter.c
index 0a9d0b9..506ffa5 100644
--- a/src/mpi/coll/iscatter.c
+++ b/src/mpi/coll/iscatter.c
@@ -463,7 +463,7 @@ int MPIR_Iscatter_inter(const void *sendbuf, int sendcount, MPI_Datatype sendtyp
         nbytes = recvtype_size * recvcount * local_size;
     }
 
-    if (nbytes < MPIR_PARAM_SCATTER_INTER_SHORT_MSG_SIZE) {
+    if (nbytes < MPIR_CVAR_SCATTER_INTER_SHORT_MSG_SIZE) {
         if (root == MPI_ROOT) {
             /* root sends all data to rank 0 on remote group and returns */
             mpi_errno = MPID_Sched_send(sendbuf, sendcount*remote_size, sendtype, 0, comm_ptr, s);
diff --git a/src/mpi/coll/red_scat.c b/src/mpi/coll/red_scat.c
index c78ec42..5a9aaca 100644
--- a/src/mpi/coll/red_scat.c
+++ b/src/mpi/coll/red_scat.c
@@ -282,7 +282,7 @@ int MPIR_Reduce_scatter_intra(const void *sendbuf, void *recvbuf, const int recv
      * a user-passed in buffer */
     MPID_Ensure_Aint_fits_in_pointer(total_count * MPIR_MAX(true_extent, extent));
 
-    if ((is_commutative) && (nbytes < MPIR_PARAM_REDSCAT_COMMUTATIVE_LONG_MSG_SIZE)) {
+    if ((is_commutative) && (nbytes < MPIR_CVAR_REDSCAT_COMMUTATIVE_LONG_MSG_SIZE)) {
         /* commutative and short. use recursive halving algorithm */
 
         /* allocate temp. buffer to receive incoming data */
@@ -505,7 +505,7 @@ int MPIR_Reduce_scatter_intra(const void *sendbuf, void *recvbuf, const int recv
         }
     }
     
-    if (is_commutative && (nbytes >= MPIR_PARAM_REDSCAT_COMMUTATIVE_LONG_MSG_SIZE)) {
+    if (is_commutative && (nbytes >= MPIR_CVAR_REDSCAT_COMMUTATIVE_LONG_MSG_SIZE)) {
 
         /* commutative and long message, or noncommutative and long message.
            use (p-1) pairwise exchanges */ 
diff --git a/src/mpi/coll/red_scat_block.c b/src/mpi/coll/red_scat_block.c
index 841435b..b9a6c5e 100644
--- a/src/mpi/coll/red_scat_block.c
+++ b/src/mpi/coll/red_scat_block.c
@@ -299,7 +299,7 @@ int MPIR_Reduce_scatter_block_intra (
      * a user-passed in buffer */
     MPID_Ensure_Aint_fits_in_pointer(total_count * MPIR_MAX(true_extent, extent));
 
-    if ((is_commutative) && (nbytes < MPIR_PARAM_REDSCAT_COMMUTATIVE_LONG_MSG_SIZE)) {
+    if ((is_commutative) && (nbytes < MPIR_CVAR_REDSCAT_COMMUTATIVE_LONG_MSG_SIZE)) {
         /* commutative and short. use recursive halving algorithm */
 
         /* allocate temp. buffer to receive incoming data */
@@ -509,7 +509,7 @@ int MPIR_Reduce_scatter_block_intra (
         }
     }
     
-    if (is_commutative && (nbytes >= MPIR_PARAM_REDSCAT_COMMUTATIVE_LONG_MSG_SIZE)) {
+    if (is_commutative && (nbytes >= MPIR_CVAR_REDSCAT_COMMUTATIVE_LONG_MSG_SIZE)) {
 
         /* commutative and long message, or noncommutative and long message.
            use (p-1) pairwise exchanges */ 
diff --git a/src/mpi/coll/reduce.c b/src/mpi/coll/reduce.c
index 4168698..0e3d7b7 100644
--- a/src/mpi/coll/reduce.c
+++ b/src/mpi/coll/reduce.c
@@ -692,7 +692,7 @@ int MPIR_Reduce_intra (
     /* check if multiple threads are calling this collective function */
     MPIDU_ERR_CHECK_MULTIPLE_THREADS_ENTER( comm_ptr );
 
-    if (MPIR_PARAM_ENABLE_SMP_COLLECTIVES && MPIR_PARAM_ENABLE_SMP_REDUCE) {
+    if (MPIR_CVAR_ENABLE_SMP_COLLECTIVES && MPIR_CVAR_ENABLE_SMP_REDUCE) {
     /* is the op commutative? We do SMP optimizations only if it is. */
     if (HANDLE_GET_KIND(op) == HANDLE_KIND_BUILTIN)
         is_commutative = 1;
@@ -702,9 +702,9 @@ int MPIR_Reduce_intra (
     }
 
     MPID_Datatype_get_size_macro(datatype, type_size);
-    nbytes = MPIR_PARAM_MAX_SMP_REDUCE_MSG_SIZE ? type_size*count : 0;
+    nbytes = MPIR_CVAR_MAX_SMP_REDUCE_MSG_SIZE ? type_size*count : 0;
     if (MPIR_Comm_is_node_aware(comm_ptr) && is_commutative &&
-        nbytes <= MPIR_PARAM_MAX_SMP_REDUCE_MSG_SIZE) {
+        nbytes <= MPIR_CVAR_MAX_SMP_REDUCE_MSG_SIZE) {
 
         void *tmp_buf = NULL;
         MPI_Aint  true_lb, true_extent, extent;
@@ -817,7 +817,7 @@ int MPIR_Reduce_intra (
     while (pof2 <= comm_size) pof2 <<= 1;
     pof2 >>=1;
 
-    if ((count*type_size > MPIR_PARAM_REDUCE_SHORT_MSG_SIZE) &&
+    if ((count*type_size > MPIR_CVAR_REDUCE_SHORT_MSG_SIZE) &&
         (HANDLE_GET_KIND(op) == HANDLE_KIND_BUILTIN) && (count >= pof2)) {
         /* do a reduce-scatter followed by gather to root. */
         mpi_errno = MPIR_Reduce_redscat_gather(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, errflag);
diff --git a/src/mpi/coll/scatter.c b/src/mpi/coll/scatter.c
index 635755b..ffd78f8 100644
--- a/src/mpi/coll/scatter.c
+++ b/src/mpi/coll/scatter.c
@@ -443,7 +443,7 @@ int MPIR_Scatter_inter(const void *sendbuf, int sendcount, MPI_Datatype sendtype
         nbytes = recvtype_size * recvcount * local_size;
     }
 
-    if (nbytes < MPIR_PARAM_SCATTER_INTER_SHORT_MSG_SIZE) {
+    if (nbytes < MPIR_CVAR_SCATTER_INTER_SHORT_MSG_SIZE) {
         if (root == MPI_ROOT) {
             /* root sends all data to rank 0 on remote group and returns */
             mpi_errno = MPIC_Send(sendbuf, sendcount*remote_size,
diff --git a/src/mpi/comm/comm_split.c b/src/mpi/comm/comm_split.c
index d53935d..379e792 100644
--- a/src/mpi/comm/comm_split.c
+++ b/src/mpi/comm/comm_split.c
@@ -67,7 +67,7 @@ static void MPIU_Sort_inttable( sorttype *keytable, int size )
 
 #if defined(HAVE_QSORT)
     /* temporary switch for profiling performance differences */
-    if (MPIR_PARAM_COMM_SPLIT_USE_QSORT)
+    if (MPIR_CVAR_COMM_SPLIT_USE_QSORT)
     {
         /* qsort isn't a stable sort, so we have to enforce stability by keeping
          * track of the original indices */
diff --git a/src/mpi/comm/commutil.c b/src/mpi/comm/commutil.c
index 4c2460e..26facb7 100644
--- a/src/mpi/comm/commutil.c
+++ b/src/mpi/comm/commutil.c
@@ -976,8 +976,8 @@ int MPIR_Get_contextid_sparse_group(MPID_Comm *comm_ptr, MPID_Group *group_ptr,
         if (eager_nelem < 0) {
             /* Ensure that at least one word of deadlock-free context IDs is
                always set aside for the base protocol */
-            MPIU_Assert( MPIR_PARAM_CTXID_EAGER_SIZE >= 0 && MPIR_PARAM_CTXID_EAGER_SIZE < MPIR_MAX_CONTEXT_MASK-1 );
-            eager_nelem = MPIR_PARAM_CTXID_EAGER_SIZE;
+            MPIU_Assert( MPIR_CVAR_CTXID_EAGER_SIZE >= 0 && MPIR_CVAR_CTXID_EAGER_SIZE < MPIR_MAX_CONTEXT_MASK-1 );
+            eager_nelem = MPIR_CVAR_CTXID_EAGER_SIZE;
         }
 
         if (ignore_id) {
diff --git a/src/mpi/debugger/dbginit.c b/src/mpi/debugger/dbginit.c
index 90a8101..5b23f74 100644
--- a/src/mpi/debugger/dbginit.c
+++ b/src/mpi/debugger/dbginit.c
@@ -173,7 +173,7 @@ void MPIR_WaitForDebugger( void )
        to access this. */
     /* Also, to avoid scaling problems, we only populate the first 64
        entries (default) */
-    maxsize = MPIR_PARAM_PROCTABLE_SIZE;
+    maxsize = MPIR_CVAR_PROCTABLE_SIZE;
     if (maxsize > size) maxsize = size;
 
     if (rank == 0) {
@@ -208,7 +208,7 @@ void MPIR_WaitForDebugger( void )
 	MPIR_proctable_size               = size;
 #if 0
 	/* Debugging hook */
-	if (MPIR_PARAM_PROCTABLE_PRINT) {
+	if (MPIR_CVAR_PROCTABLE_PRINT) {
 	    for (i=0; i<maxsize; i++) {
 		printf( "PT[%d].pid = %d, .host_name = %s\n", 
 			i, MPIR_proctable[i].pid, MPIR_proctable[i].host_name );
diff --git a/src/mpi/errhan/errnames.txt b/src/mpi/errhan/errnames.txt
index 4ce70c6..2641b5d 100644
--- a/src/mpi/errhan/errnames.txt
+++ b/src/mpi/errhan/errnames.txt
@@ -876,9 +876,9 @@ is too big (> MPIU_SHMW_GHND_SZ)
 
 **toomanynbc: too many outstanding nonblocking collectives detected
 
-**ifname_and_hostname:Only one of MPIR_PARAM_CH3_INTERFACE_HOSTNAME and MPIR_PARAM_CH3_NETWORK_IFACE environment variables are allowed, but both were set.
-**iface_notfound:The network interface specified in MPIR_PARAM_CH3_NETWORK_IFACE was not found.
-**iface_notfound %s:The network interface, \"%s\", specified in MPIR_PARAM_CH3_NETWORK_IFACE was not found.
+**ifname_and_hostname:Only one of MPIR_CVAR_CH3_INTERFACE_HOSTNAME and MPIR_CVAR_CH3_NETWORK_IFACE environment variables are allowed, but both were set.
+**iface_notfound:The network interface specified in MPIR_CVAR_CH3_NETWORK_IFACE was not found.
+**iface_notfound %s:The network interface, \"%s\", specified in MPIR_CVAR_CH3_NETWORK_IFACE was not found.
 
 **procnamefailed:Failed to get processor name
 
diff --git a/src/mpi/errhan/errutil.c b/src/mpi/errhan/errutil.c
index 0420fb1..dc275c3 100644
--- a/src/mpi/errhan/errutil.c
+++ b/src/mpi/errhan/errutil.c
@@ -1153,7 +1153,7 @@ static void MPIR_Err_print_stack_string(int errcode, char *str, int maxlen )
 		    maxlen--;
 		}
 		
-		if (MPIR_PARAM_CHOP_ERROR_STACK > 0)
+		if (MPIR_CVAR_CHOP_ERROR_STACK > 0)
 		{
 		    cur_pos = ErrorRing[ring_idx].msg;
 		    len = (int)strlen(cur_pos);
@@ -1163,16 +1163,16 @@ static void MPIR_Err_print_stack_string(int errcode, char *str, int maxlen )
 		    }
 		    while (len)
 		    {
-			if (len >= MPIR_PARAM_CHOP_ERROR_STACK - max_location_len)
+			if (len >= MPIR_CVAR_CHOP_ERROR_STACK - max_location_len)
 			{
 			    if (len > maxlen)
 				break;
 			    /* FIXME: Don't use Snprint to append a string ! */
-			    MPIU_Snprintf(str, MPIR_PARAM_CHOP_ERROR_STACK - 1 - max_location_len, "%s", cur_pos);
-			    str[MPIR_PARAM_CHOP_ERROR_STACK - 1 - max_location_len] = '\n';
-			    cur_pos += MPIR_PARAM_CHOP_ERROR_STACK - 1 - max_location_len;
-			    str += MPIR_PARAM_CHOP_ERROR_STACK - max_location_len;
-			    maxlen -= MPIR_PARAM_CHOP_ERROR_STACK - max_location_len;
+			    MPIU_Snprintf(str, MPIR_CVAR_CHOP_ERROR_STACK - 1 - max_location_len, "%s", cur_pos);
+			    str[MPIR_CVAR_CHOP_ERROR_STACK - 1 - max_location_len] = '\n';
+			    cur_pos += MPIR_CVAR_CHOP_ERROR_STACK - 1 - max_location_len;
+			    str += MPIR_CVAR_CHOP_ERROR_STACK - max_location_len;
+			    maxlen -= MPIR_CVAR_CHOP_ERROR_STACK - max_location_len;
 			    if (maxlen < max_location_len)
 				break;
 			    for (i=0; i<max_location_len; i++)
@@ -1777,8 +1777,8 @@ static void MPIR_Err_stack_init( void )
 
     error_ring_mutex_create(&mpi_errno);
 
-    if (MPIR_PARAM_CHOP_ERROR_STACK < 0) {
-        MPIR_PARAM_CHOP_ERROR_STACK = 80;
+    if (MPIR_CVAR_CHOP_ERROR_STACK < 0) {
+        MPIR_CVAR_CHOP_ERROR_STACK = 80;
 #ifdef HAVE_WINDOWS_H
         {
             /* If windows, set the default width to the window size */
@@ -1789,7 +1789,7 @@ static void MPIR_Err_stack_init( void )
                 if (GetConsoleScreenBufferInfo(hConsole, &info))
                 {
                     /* override the parameter system in this case */
-                    MPIR_PARAM_CHOP_ERROR_STACK = info.dwMaximumWindowSize.X;
+                    MPIR_CVAR_CHOP_ERROR_STACK = info.dwMaximumWindowSize.X;
                 }
             }
         }
@@ -1964,7 +1964,7 @@ static int ErrGetInstanceString( int errorcode, char msg[], int num_remaining )
 {
     int len;
 
-    if (MPIR_PARAM_PRINT_ERROR_STACK) {
+    if (MPIR_CVAR_PRINT_ERROR_STACK) {
 	MPIU_Strncpy(msg, ", error stack:\n", num_remaining);
 	msg[num_remaining - 1] = '\0';
 	len = (int)strlen(msg);
diff --git a/src/mpi/init/finalize.c b/src/mpi/init/finalize.c
index 91e4c47..19f3d9f 100644
--- a/src/mpi/init/finalize.c
+++ b/src/mpi/init/finalize.c
@@ -243,7 +243,7 @@ int MPI_Finalize( void )
        go to separate files or to be sorted by rank (note that
        the rank is at the head of the line) */
     {
-	if (MPIR_PARAM_MEMDUMP) {
+	if (MPIR_CVAR_MEMDUMP) {
 	    /* The second argument is the min id to print; memory allocated 
 	       after MPI_Init is given an id of one.  This allows us to
 	       ignore, if desired, memory leaks in the MPID_Init call */
diff --git a/src/mpi/init/init.c b/src/mpi/init/init.c
index 3d7789d..be70c9e 100644
--- a/src/mpi/init/init.c
+++ b/src/mpi/init/init.c
@@ -103,28 +103,28 @@ int MPI_Init( int *argc, char ***argv )
 
     MPIR_T_env_init();
 
-    if (!strcmp(MPIR_PARAM_DEFAULT_THREAD_LEVEL, "MPI_THREAD_MULTIPLE"))
+    if (!strcmp(MPIR_CVAR_DEFAULT_THREAD_LEVEL, "MPI_THREAD_MULTIPLE"))
         threadLevel = MPI_THREAD_MULTIPLE;
-    else if (!strcmp(MPIR_PARAM_DEFAULT_THREAD_LEVEL, "MPI_THREAD_SERIALIZED"))
+    else if (!strcmp(MPIR_CVAR_DEFAULT_THREAD_LEVEL, "MPI_THREAD_SERIALIZED"))
         threadLevel = MPI_THREAD_SERIALIZED;
-    else if (!strcmp(MPIR_PARAM_DEFAULT_THREAD_LEVEL, "MPI_THREAD_FUNNELED"))
+    else if (!strcmp(MPIR_CVAR_DEFAULT_THREAD_LEVEL, "MPI_THREAD_FUNNELED"))
         threadLevel = MPI_THREAD_FUNNELED;
-    else if (!strcmp(MPIR_PARAM_DEFAULT_THREAD_LEVEL, "MPI_THREAD_SINGLE"))
+    else if (!strcmp(MPIR_CVAR_DEFAULT_THREAD_LEVEL, "MPI_THREAD_SINGLE"))
         threadLevel = MPI_THREAD_SINGLE;
     else {
-        MPIU_Error_printf("Unrecognized thread level %s\n", MPIR_PARAM_DEFAULT_THREAD_LEVEL);
+        MPIU_Error_printf("Unrecognized thread level %s\n", MPIR_CVAR_DEFAULT_THREAD_LEVEL);
         exit(1);
     }
 
     /* If the user requested for asynchronous progress, request for
      * THREAD_MULTIPLE. */
-    if (MPIR_PARAM_ASYNC_PROGRESS)
+    if (MPIR_CVAR_ASYNC_PROGRESS)
         threadLevel = MPI_THREAD_MULTIPLE;
 
     mpi_errno = MPIR_Init_thread( argc, argv, threadLevel, &provided );
     if (mpi_errno != MPI_SUCCESS) goto fn_fail;
 
-    if (MPIR_PARAM_ASYNC_PROGRESS) {
+    if (MPIR_CVAR_ASYNC_PROGRESS) {
         if (provided == MPI_THREAD_MULTIPLE) {
             mpi_errno = MPIR_Init_async_thread();
             if (mpi_errno) goto fn_fail;
diff --git a/src/mpi/init/initthread.c b/src/mpi/init/initthread.c
index b15c66c..cd9f159 100644
--- a/src/mpi/init/initthread.c
+++ b/src/mpi/init/initthread.c
@@ -391,7 +391,7 @@ int MPIR_Init_thread(int * argc, char ***argv, int required, int * provided)
     /* MPIU_Timer_pre_init(); */
 
     /* Wait for debugger to attach if requested. */
-    if (MPIR_PARAM_DEBUG_HOLD) {
+    if (MPIR_CVAR_DEBUG_HOLD) {
         volatile int hold = 1;
         while (hold)
 #ifdef HAVE_USLEEP
@@ -402,7 +402,7 @@ int MPIR_Init_thread(int * argc, char ***argv, int required, int * provided)
 
 
 #if defined(HAVE_ERROR_CHECKING) && (HAVE_ERROR_CHECKING == MPID_ERROR_LEVEL_RUNTIME)
-    MPIR_Process.do_error_checks = MPIR_PARAM_ERROR_CHECKING;
+    MPIR_Process.do_error_checks = MPIR_CVAR_ERROR_CHECKING;
 #endif
 
     /* define MPI as initialized so that we can use MPI functions within 
@@ -586,13 +586,13 @@ int MPI_Init_thread( int *argc, char ***argv, int required, int *provided )
 
     /* If the user requested for asynchronous progress, request for
      * THREAD_MULTIPLE. */
-    if (MPIR_PARAM_ASYNC_PROGRESS)
+    if (MPIR_CVAR_ASYNC_PROGRESS)
         reqd = MPI_THREAD_MULTIPLE;
 
     mpi_errno = MPIR_Init_thread( argc, argv, reqd, provided );
     if (mpi_errno != MPI_SUCCESS) goto fn_fail;
 
-    if (MPIR_PARAM_ASYNC_PROGRESS) {
+    if (MPIR_CVAR_ASYNC_PROGRESS) {
         if (*provided == MPI_THREAD_MULTIPLE) {
             mpi_errno = MPIR_Init_async_thread();
             if (mpi_errno) goto fn_fail;
diff --git a/src/mpi_t/mpit_finalize.c b/src/mpi_t/mpit_finalize.c
index 23c3559..ea33a4e 100644
--- a/src/mpi_t/mpit_finalize.c
+++ b/src/mpi_t/mpit_finalize.c
@@ -92,7 +92,7 @@ static void MPIR_T_cvar_env_finalize(void)
     int i;
     cvar_table_entry_t *cvar;
 
-    MPIR_Param_finalize();
+    MPIR_T_cvar_finalize();
 
     if (cvar_table) {
         /* Free all entries */
diff --git a/src/mpi_t/mpit_initthread.c b/src/mpi_t/mpit_initthread.c
index 31f0ec5..02aed27 100644
--- a/src/mpi_t/mpit_initthread.c
+++ b/src/mpi_t/mpit_initthread.c
@@ -48,7 +48,7 @@ static inline void MPIR_T_cvar_env_init(void)
 
     utarray_new(cvar_table, &cvar_table_entry_icd);
     cvar_hash = NULL;
-    MPIR_Param_init_params();
+    MPIR_T_cvar_init();
 }
 
 static inline void MPIR_T_pvar_env_init(void)
diff --git a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_init.c b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_init.c
index 9a9e863..e0b83e4 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_init.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/portals4/ptl_init.c
@@ -281,7 +281,7 @@ static int vc_init(MPIDI_VC_t *vc)
     vc_ch->iStartContigMsg = MPID_nem_ptl_iStartContigMsg;
     vc_ch->iSendContig     = MPID_nem_ptl_iSendContig;
 
-    if (MPIR_PARAM_COMM_OVERRIDES) /* allow feature to be disabled at runtime */
+    if (MPIR_CVAR_COMM_OVERRIDES) /* allow feature to be disabled at runtime */
         vc->comm_ops = &comm_ops;
 
     vc_ch->next = NULL;
diff --git a/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_init.c b/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_init.c
index 6c4e4fb..02f7c2b 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_init.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/tcp/tcp_init.c
@@ -234,16 +234,16 @@ static int GetSockInterfaceAddr(int myRank, char *ifname, int maxIfname,
     MPIU_Assert(maxIfname);
     ifname[0] = '\0';
 
-    MPIU_ERR_CHKANDJUMP(MPIR_PARAM_CH3_INTERFACE_HOSTNAME && MPIR_PARAM_NEMESIS_TCP_NETWORK_IFACE, mpi_errno, MPI_ERR_OTHER, "**ifname_and_hostname");
+    MPIU_ERR_CHKANDJUMP(MPIR_CVAR_CH3_INTERFACE_HOSTNAME && MPIR_CVAR_NEMESIS_TCP_NETWORK_IFACE, mpi_errno, MPI_ERR_OTHER, "**ifname_and_hostname");
     
     /* Set "not found" for ifaddr */
     ifaddr->len = 0;
 
     /* Check if user specified ethernet interface name, e.g., ib0, eth1 */
-    if (MPIR_PARAM_NEMESIS_TCP_NETWORK_IFACE) {
+    if (MPIR_CVAR_NEMESIS_TCP_NETWORK_IFACE) {
 	int len;
-        mpi_errno = MPIDI_Get_IP_for_iface(MPIR_PARAM_NEMESIS_TCP_NETWORK_IFACE, ifaddr, &ifaddrFound);
-        MPIU_ERR_CHKANDJUMP1(mpi_errno || !ifaddrFound, mpi_errno, MPI_ERR_OTHER, "**iface_notfound", "**iface_notfound %s", MPIR_PARAM_NEMESIS_TCP_NETWORK_IFACE);
+        mpi_errno = MPIDI_Get_IP_for_iface(MPIR_CVAR_NEMESIS_TCP_NETWORK_IFACE, ifaddr, &ifaddrFound);
+        MPIU_ERR_CHKANDJUMP1(mpi_errno || !ifaddrFound, mpi_errno, MPI_ERR_OTHER, "**iface_notfound", "**iface_notfound %s", MPIR_CVAR_NEMESIS_TCP_NETWORK_IFACE);
         
         MPIU_DBG_MSG_FMT(CH3_CONNECT, VERBOSE, (MPIU_DBG_FDEST,
                                                 "ifaddrFound=TRUE ifaddr->type=%d ifaddr->len=%d ifaddr->ifaddr[0-3]=%#08x",
@@ -256,7 +256,7 @@ static int GetSockInterfaceAddr(int myRank, char *ifname, int maxIfname,
     }
 
     /* Check for a host name supplied through an environment variable */
-    ifname_string = MPIR_PARAM_CH3_INTERFACE_HOSTNAME;
+    ifname_string = MPIR_CVAR_CH3_INTERFACE_HOSTNAME;
     if (!ifname_string) {
 	/* See if there is a per-process name for the interfaces (e.g.,
 	   the process manager only delievers the same values for the 
@@ -305,7 +305,7 @@ static int GetSockInterfaceAddr(int myRank, char *ifname, int maxIfname,
     if (!ifaddrFound) {
         int i;
 	struct hostent *info = NULL;
-        for (i = 0; i < MPIR_PARAM_NEMESIS_TCP_HOST_LOOKUP_RETRIES; ++i) {
+        for (i = 0; i < MPIR_CVAR_NEMESIS_TCP_HOST_LOOKUP_RETRIES; ++i) {
             info = gethostbyname( ifname_string );
             if (info || h_errno != TRY_AGAIN)
                 break;
@@ -558,11 +558,11 @@ int MPID_nem_tcp_bind (int sockfd)
 
     MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_BIND);
    
-    MPIU_ERR_CHKANDJUMP(MPIR_PARAM_CH3_PORT_RANGE.low < 0 || MPIR_PARAM_CH3_PORT_RANGE.low > MPIR_PARAM_CH3_PORT_RANGE.high, mpi_errno, MPI_ERR_OTHER, "**badportrange");
+    MPIU_ERR_CHKANDJUMP(MPIR_CVAR_CH3_PORT_RANGE.low < 0 || MPIR_CVAR_CH3_PORT_RANGE.low > MPIR_CVAR_CH3_PORT_RANGE.high, mpi_errno, MPI_ERR_OTHER, "**badportrange");
 
     /* default MPICH_PORT_RANGE is {0,0} so bind will use any available port */
     ret = 0;
-    for (port = MPIR_PARAM_CH3_PORT_RANGE.low; port <= MPIR_PARAM_CH3_PORT_RANGE.high; ++port)
+    for (port = MPIR_CVAR_CH3_PORT_RANGE.low; port <= MPIR_CVAR_CH3_PORT_RANGE.high; ++port)
     {
         memset ((void *)&sin, 0, sizeof(sin));
         sin.sin_family      = AF_INET;
diff --git a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_init.c b/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_init.c
index 0d04c1d..f42d7b2 100644
--- a/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_init.c
+++ b/src/mpid/ch3/channels/nemesis/netmod/wintcp/wintcp_init.c
@@ -421,13 +421,13 @@ int MPID_nem_newtcp_module_bind (MPIU_SOCKW_Sockfd_t sockfd)
 
     MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWTCP_MODULE_BIND);
    
-    MPIU_ERR_CHKANDJUMP(MPIR_PARAM_CH3_PORT_RANGE.low < 0 || MPIR_PARAM_CH3_PORT_RANGE.low > MPIR_PARAM_CH3_PORT_RANGE.high, mpi_errno, MPI_ERR_OTHER, "**badportrange");
+    MPIU_ERR_CHKANDJUMP(MPIR_CVAR_CH3_PORT_RANGE.low < 0 || MPIR_CVAR_CH3_PORT_RANGE.low > MPIR_CVAR_CH3_PORT_RANGE.high, mpi_errno, MPI_ERR_OTHER, "**badportrange");
 
     memset((void *)&sin, 0, sizeof(sin));
     sin.sin_family      = AF_INET;
     sin.sin_addr.s_addr = htonl(INADDR_ANY);
 
-    mpi_errno = MPIU_SOCKW_Bind_port_range(sockfd, &sin, MPIR_PARAM_CH3_PORT_RANGE.low, MPIR_PARAM_CH3_PORT_RANGE.high);
+    mpi_errno = MPIU_SOCKW_Bind_port_range(sockfd, &sin, MPIR_CVAR_CH3_PORT_RANGE.low, MPIR_CVAR_CH3_PORT_RANGE.high);
     if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
 
  fn_exit:
diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_progress.c b/src/mpid/ch3/channels/nemesis/src/ch3_progress.c
index 4535325..cbabb05 100644
--- a/src/mpid/ch3/channels/nemesis/src/ch3_progress.c
+++ b/src/mpid/ch3/channels/nemesis/src/ch3_progress.c
@@ -310,7 +310,7 @@ int MPIDI_CH3I_Progress (MPID_Progress_state *progress_state, int is_blocking)
     }
     
 #ifdef ENABLE_CHECKPOINTING
-    if (MPIR_PARAM_NEMESIS_ENABLE_CKPOINT) {
+    if (MPIR_CVAR_NEMESIS_ENABLE_CKPOINT) {
         if (MPIDI_nem_ckpt_start_checkpoint) {
             MPIDI_nem_ckpt_start_checkpoint = FALSE;
             mpi_errno = MPIDI_nem_ckpt_start();
diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_ckpt.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_ckpt.c
index 1f74e85..96a6df1 100644
--- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_ckpt.c
+++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_ckpt.c
@@ -146,7 +146,7 @@ int MPIDI_nem_ckpt_init(void)
 
     MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_NEM_CKPT_INIT);
 
-    if (!MPIR_PARAM_NEMESIS_ENABLE_CKPOINT)
+    if (!MPIR_CVAR_NEMESIS_ENABLE_CKPOINT)
         goto fn_exit;
     
     client_id = cr_init();
diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c
index 80e1556..5674e8d 100644
--- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c
+++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c
@@ -506,15 +506,15 @@ MPID_nem_vc_init (MPIDI_VC_t *vc)
         vc_ch->lmt_active_lmt      = NULL;
         vc_ch->lmt_enqueued        = FALSE;
 
-        if (MPIR_PARAM_NEMESIS_SHM_EAGER_MAX_SZ == -1)
+        if (MPIR_CVAR_NEMESIS_SHM_EAGER_MAX_SZ == -1)
             vc->eager_max_msg_sz = MPID_NEM_MPICH_DATA_LEN - sizeof(MPIDI_CH3_Pkt_t);
         else
-            vc->eager_max_msg_sz = MPIR_PARAM_NEMESIS_SHM_EAGER_MAX_SZ;
+            vc->eager_max_msg_sz = MPIR_CVAR_NEMESIS_SHM_EAGER_MAX_SZ;
 
-        if (MPIR_PARAM_NEMESIS_SHM_READY_EAGER_MAX_SZ == -2)
+        if (MPIR_CVAR_NEMESIS_SHM_READY_EAGER_MAX_SZ == -2)
             vc->ready_eager_max_msg_sz = vc->eager_max_msg_sz; /* force local ready sends to use LMT */
         else
-            vc->ready_eager_max_msg_sz = MPIR_PARAM_NEMESIS_SHM_READY_EAGER_MAX_SZ;
+            vc->ready_eager_max_msg_sz = MPIR_CVAR_NEMESIS_SHM_READY_EAGER_MAX_SZ;
 
         MPIU_DBG_MSG(VC, VERBOSE, "vc using shared memory");
     }
diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_dma.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_dma.c
index 3e1b487..7fff986 100644
--- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_dma.c
+++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_lmt_dma.c
@@ -355,7 +355,7 @@ int MPID_nem_lmt_dma_start_recv(MPIDI_VC_t *vc, MPID_Request *rreq, MPID_IOV s_c
     MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype,
                             dt_contig, data_sz, dt_ptr, dt_true_lb);
 
-    nodma = !knem_has_dma || data_sz < MPIR_PARAM_NEM_LMT_DMA_THRESHOLD;
+    nodma = !knem_has_dma || data_sz < MPIR_CVAR_NEM_LMT_DMA_THRESHOLD;
 
     if (dt_contig) {
         /* handle the iov creation ourselves */
diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_network.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_network.c
index a8d7bd4..fb93e43 100644
--- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_network.c
+++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_network.c
@@ -27,8 +27,8 @@ int MPID_nem_choose_netmod(void)
 
     MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_CHOOSE_NETMOD);
 
-    MPIU_Assert(MPIR_PARAM_NEMESIS_NETMOD != NULL);
-    if (strcmp(MPIR_PARAM_NEMESIS_NETMOD, "") == 0)
+    MPIU_Assert(MPIR_CVAR_NEMESIS_NETMOD != NULL);
+    if (strcmp(MPIR_CVAR_NEMESIS_NETMOD, "") == 0)
     {
         /* netmod not specified, using the default */
         MPID_nem_netmod_func = MPID_nem_netmod_funcs[0];
@@ -41,7 +41,7 @@ int MPID_nem_choose_netmod(void)
 
     for (i = 0; i < MPID_nem_num_netmods; ++i)
     {
-        if (!MPIU_Strncasecmp(MPIR_PARAM_NEMESIS_NETMOD, MPID_nem_netmod_strings[i], MPID_NEM_MAX_NETMOD_STRING_LEN))
+        if (!MPIU_Strncasecmp(MPIR_CVAR_NEMESIS_NETMOD, MPID_nem_netmod_strings[i], MPID_NEM_MAX_NETMOD_STRING_LEN))
         {
             MPID_nem_netmod_func = MPID_nem_netmod_funcs[i];
             MPID_nem_netmod_id = i;
@@ -52,7 +52,7 @@ int MPID_nem_choose_netmod(void)
         }
     }
 
-    MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**invalid_netmod", "**invalid_netmod %s", MPIR_PARAM_NEMESIS_NETMOD);
+    MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**invalid_netmod", "**invalid_netmod %s", MPIR_CVAR_NEMESIS_NETMOD);
 
  fn_exit:
     MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_CHOOSE_NETMOD);
diff --git a/src/mpid/ch3/src/ch3u_rma_sync.c b/src/mpid/ch3/src/ch3u_rma_sync.c
index da8ae8b..0d51fc1 100644
--- a/src/mpid/ch3/src/ch3u_rma_sync.c
+++ b/src/mpid/ch3/src/ch3u_rma_sync.c
@@ -385,8 +385,8 @@ int MPIDI_Win_fence(int assert, MPID_Win *win_ptr)
 		   this, significant overhead is added once the
 		   number of requests exceeds the threshold, since the
 		   number that are completed in a call may be small. */
-		if (nRequest > MPIR_PARAM_CH3_RMA_NREQUEST_THRESHOLD &&
-		    nRequest - nRequestNew > MPIR_PARAM_CH3_RMA_NREQUEST_NEW_THRESHOLD) {
+		if (nRequest > MPIR_CVAR_CH3_RMA_NREQUEST_THRESHOLD &&
+		    nRequest - nRequestNew > MPIR_CVAR_CH3_RMA_NREQUEST_NEW_THRESHOLD) {
 		    int nDone = 0;
 		    MPIU_INSTR_STMT(list_complete=MPIU_INSTR_GET_VAR(winfence_complete));
                     MPIDI_CH3I_RMAListPartialComplete(win_ptr, ops_list, curr_ptr, &nDone);
@@ -879,7 +879,7 @@ static int MPIDI_CH3I_Send_contig_acc_msg(MPIDI_RMA_Op_t *rma_op,
     MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size);
     /* FIXME: Make this size check efficient and match the packet type */
     len = rma_op->origin_count * origin_type_size;
-    if (MPIR_PARAM_CH3_RMA_ACC_IMMED && len <= MPIDI_RMA_IMMED_INTS*sizeof(int)) {
+    if (MPIR_CVAR_CH3_RMA_ACC_IMMED && len <= MPIDI_RMA_IMMED_INTS*sizeof(int)) {
 	MPIDI_CH3_Pkt_accum_immed_t * accumi_pkt = &upkt.accum_immed;
 	void *dest = accumi_pkt->data, *src = rma_op->origin_addr;
 	
@@ -1667,8 +1667,8 @@ int MPIDI_Win_complete(MPID_Win *win_ptr)
 	    nRequest++;
 	    MPIU_INSTR_COUNTER_INCR(wincomplete_reqs,1);
 	    curr_ptr    = curr_ptr->next;
-	    if (nRequest > MPIR_PARAM_CH3_RMA_NREQUEST_THRESHOLD &&
-		nRequest - nRequestNew > MPIR_PARAM_CH3_RMA_NREQUEST_NEW_THRESHOLD) {
+	    if (nRequest > MPIR_CVAR_CH3_RMA_NREQUEST_THRESHOLD &&
+		nRequest - nRequestNew > MPIR_CVAR_CH3_RMA_NREQUEST_NEW_THRESHOLD) {
 		int nDone = 0;
 		MPIU_INSTR_STMT(list_complete=MPIU_INSTR_GET_VAR(wincomplete_complete));
                 MPIDI_CH3I_RMAListPartialComplete(win_ptr, ops_list, curr_ptr, &nDone);
@@ -1947,7 +1947,7 @@ int MPIDI_Win_lock(int lock_type, int dest, int assert, MPID_Win *win_ptr)
             if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
         }
     }
-    else if (MPIR_PARAM_CH3_RMA_LOCK_IMMED && ((assert & MPI_MODE_NOCHECK) == 0)) {
+    else if (MPIR_CVAR_CH3_RMA_LOCK_IMMED && ((assert & MPI_MODE_NOCHECK) == 0)) {
         /* TODO: Make this mode of operation available through an assert
            argument or info key. */
         mpi_errno = MPIDI_CH3I_Send_lock_msg(dest, lock_type, win_ptr);
@@ -2029,7 +2029,7 @@ int MPIDI_Win_unlock(int dest, MPID_Win *win_ptr)
     /* TODO: MPI-3: Add lock_all->op optimization. */
     /* LOCK-OP-UNLOCK Optimization -- This optimization can't be used if we
        have already requested the lock. */
-    if ( MPIR_PARAM_CH3_RMA_MERGE_LOCK_OP_UNLOCK &&
+    if ( MPIR_CVAR_CH3_RMA_MERGE_LOCK_OP_UNLOCK &&
          win_ptr->targets[dest].remote_lock_state == MPIDI_CH3_WIN_LOCK_CALLED &&
          rma_op && rma_op->next == NULL /* There is only one op */ &&
          rma_op->type != MPIDI_RMA_COMPARE_AND_SWAP &&
@@ -2688,8 +2688,8 @@ static int MPIDI_CH3I_Do_passive_target_rma(MPID_Win *win_ptr, int target_rank,
 	    nRequest++;
 	    MPIU_INSTR_COUNTER_INCR(winunlock_reqs,1);
 	    curr_ptr    = curr_ptr->next;
-	    if (nRequest > MPIR_PARAM_CH3_RMA_NREQUEST_THRESHOLD &&
-		nRequest - nRequestNew > MPIR_PARAM_CH3_RMA_NREQUEST_NEW_THRESHOLD) {
+	    if (nRequest > MPIR_CVAR_CH3_RMA_NREQUEST_THRESHOLD &&
+		nRequest - nRequestNew > MPIR_CVAR_CH3_RMA_NREQUEST_NEW_THRESHOLD) {
 		int nDone = 0;
 		MPIU_INSTR_STMT(list_complete=MPIU_INSTR_GET_VAR(winunlock_complete));
                 MPIDI_CH3I_RMAListPartialComplete(win_ptr,
diff --git a/src/mpid/ch3/src/mpid_vc.c b/src/mpid/ch3/src/mpid_vc.c
index cf511bd..73b4796 100644
--- a/src/mpid/ch3/src/mpid_vc.c
+++ b/src/mpid/ch3/src/mpid_vc.c
@@ -722,7 +722,7 @@ int MPIDI_VC_Init( MPIDI_VC_t *vc, MPIDI_PG_t *pg, int rank )
     vc->rndvSend_fn      = MPIDI_CH3_RndvSend;
     vc->rndvRecv_fn      = MPIDI_CH3_RecvRndv;
     vc->ready_eager_max_msg_sz = -1; /* no limit */;
-    vc->eager_max_msg_sz = MPIR_PARAM_CH3_EAGER_MAX_MSG_SIZE;
+    vc->eager_max_msg_sz = MPIR_CVAR_CH3_EAGER_MAX_MSG_SIZE;
 
     vc->sendNoncontig_fn = MPIDI_CH3_SendNoncontig_iov;
 #ifdef ENABLE_COMM_OVERRIDES
@@ -1120,7 +1120,7 @@ int MPIDI_Populate_vc_node_ids(MPIDI_PG_t *pg, int our_pg_rank)
 #ifdef ENABLED_NO_LOCAL
     no_local = 1;
 #else
-    no_local = MPIR_PARAM_CH3_NOLOCAL;
+    no_local = MPIR_CVAR_CH3_NOLOCAL;
 #endif
 
     /* Used for debugging on a single machine: Odd procs on a node are
@@ -1129,7 +1129,7 @@ int MPIDI_Populate_vc_node_ids(MPIDI_PG_t *pg, int our_pg_rank)
 #ifdef ENABLED_ODD_EVEN_CLIQUES
     odd_even_cliques = 1;
 #else
-    odd_even_cliques = MPIR_PARAM_CH3_ODD_EVEN_CLIQUES;
+    odd_even_cliques = MPIR_CVAR_CH3_ODD_EVEN_CLIQUES;
 #endif
 
     if (no_local) {
diff --git a/src/mpid/common/sock/iocp/sock.c b/src/mpid/common/sock/iocp/sock.c
index e838d71..51e6c15 100644
--- a/src/mpid/common/sock/iocp/sock.c
+++ b/src/mpid/common/sock/iocp/sock.c
@@ -142,11 +142,11 @@ static int easy_create_ranged(SOCKET *sock, int port, unsigned long addr)
 	return mpi_errno;
     }
     
-    MPIU_ERR_CHKANDJUMP(MPIR_PARAM_CH3_PORT_RANGE.low < 0 || MPIR_PARAM_CH3_PORT_RANGE.low > MPIR_PARAM_CH3_PORT_RANGE.high, mpi_errno, MPI_ERR_OTHER, "**badportrange");
-    if (port == 0 && MPIR_PARAM_CH3_PORT_RANGE.low != 0 && MPIR_PARAM_CH3_PORT_RANGE.high != 0)
+    MPIU_ERR_CHKANDJUMP(MPIR_CVAR_CH3_PORT_RANGE.low < 0 || MPIR_CVAR_CH3_PORT_RANGE.low > MPIR_CVAR_CH3_PORT_RANGE.high, mpi_errno, MPI_ERR_OTHER, "**badportrange");
+    if (port == 0 && MPIR_CVAR_CH3_PORT_RANGE.low != 0 && MPIR_CVAR_CH3_PORT_RANGE.high != 0)
     {
 	use_range = 1;
-	port = MPIR_PARAM_CH3_PORT_RANGE.low;
+	port = MPIR_CVAR_CH3_PORT_RANGE.low;
     }
 
     memset(&sockAddr,0,sizeof(sockAddr));
@@ -162,7 +162,7 @@ static int easy_create_ranged(SOCKET *sock, int port, unsigned long addr)
 	    if (use_range)
 	    {
 		port++;
-		if (port > MPIR_PARAM_CH3_PORT_RANGE.high)
+		if (port > MPIR_CVAR_CH3_PORT_RANGE.high)
 		{
 		    mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPIDU_SOCK_ERR_FAIL, "**socket", 0);
 		    return mpi_errno;
diff --git a/src/mpid/pamid/src/mpidi_env.c b/src/mpid/pamid/src/mpidi_env.c
index b85c40d..732140a 100644
--- a/src/mpid/pamid/src/mpidi_env.c
+++ b/src/mpid/pamid/src/mpidi_env.c
@@ -69,7 +69,7 @@
  *   libraries (gcc, xl, xl.ndebug) and MPI_Init_thread() is called with
  *   MPI_THREAD_MULTIPLE.
  *   - NOTE: This environment variable has the same effect as setting
- *           MPIR_PARAM_DEFAULT_THREAD_LEVEL=multiple
+ *           MPIR_CVAR_DEFAULT_THREAD_LEVEL=multiple
  *
  * - PAMID_CONTEXT_MAX - This variable sets the maximum allowable number
  *   of contexts. Contexts are a method of dividing hardware resources
diff --git a/src/nameserv/file/file_nameserv.c b/src/nameserv/file/file_nameserv.c
index 9be8af9..97e45fa 100644
--- a/src/nameserv/file/file_nameserv.c
+++ b/src/nameserv/file/file_nameserv.c
@@ -57,7 +57,7 @@ int MPID_NS_Create( const MPID_Info *info_ptr, MPID_NS_Handle *handle_ptr )
     (*handle_ptr)->mypid   = getpid();
 
     /* Get the dirname.  Could use an info value of NAMEPUB_CONTACT */
-    dirname = MPIR_PARAM_NAMESERV_FILE_PUBDIR;
+    dirname = MPIR_CVAR_NAMESERV_FILE_PUBDIR;
     if (!dirname) {
         /* user did not specify a directory, try using HOME */
         ret = MPL_env2str("HOME", &dirname);
diff --git a/src/pm/hydra/ui/mpich/mpiexec.c b/src/pm/hydra/ui/mpich/mpiexec.c
index 0e80405..b38b2c6 100644
--- a/src/pm/hydra/ui/mpich/mpiexec.c
+++ b/src/pm/hydra/ui/mpich/mpiexec.c
@@ -305,7 +305,7 @@ int main(int argc, char **argv)
      * range. */
     if (MPL_env2str("MPIEXEC_PORTRANGE", (const char **) &HYD_server_info.port_range) ||
         MPL_env2str("MPIEXEC_PORT_RANGE", (const char **) &HYD_server_info.port_range) ||
-        MPL_env2str("MPIR_PARAM_CH3_PORT_RANGE", (const char **) &HYD_server_info.port_range))
+        MPL_env2str("MPIR_CVAR_CH3_PORT_RANGE", (const char **) &HYD_server_info.port_range))
         HYD_server_info.port_range = HYDU_strdup(HYD_server_info.port_range);
 
     /* Add the stdout/stderr callback handlers */
diff --git a/src/pm/hydra/ui/mpich/utils.c b/src/pm/hydra/ui/mpich/utils.c
index 2a10515..9d55fd4 100644
--- a/src/pm/hydra/ui/mpich/utils.c
+++ b/src/pm/hydra/ui/mpich/utils.c
@@ -1541,7 +1541,7 @@ static HYD_status set_default_values(void)
                                 "cannot set iface and force hostname propagation");
         }
 
-        HYDU_append_env_to_list("MPIR_PARAM_NEMESIS_TCP_NETWORK_IFACE",
+        HYDU_append_env_to_list("MPIR_CVAR_NEMESIS_TCP_NETWORK_IFACE",
                                 HYD_server_info.user_global.iface,
                                 &HYD_server_info.user_global.global_env.system);
 
@@ -1552,7 +1552,7 @@ static HYD_status set_default_values(void)
     /* If hostname propagation is requested (or not set), set the
      * environment variable for doing that */
     if (hostname_propagation || hostname_propagation == -1)
-        HYD_server_info.iface_ip_env_name = HYDU_strdup("MPIR_PARAM_CH3_INTERFACE_HOSTNAME");
+        HYD_server_info.iface_ip_env_name = HYDU_strdup("MPIR_CVAR_CH3_INTERFACE_HOSTNAME");
 
     /* Default universe size if the user did not specify anything is
      * INFINITE */
@@ -1742,7 +1742,7 @@ HYD_status HYD_uii_mpx_get_parameters(char **t_argv)
     /* If the user set the checkpoint prefix, set env var to enable
      * checkpointing on the processes  */
     if (HYD_server_info.user_global.ckpoint_prefix)
-        HYDU_append_env_to_list("MPIR_PARAM_NEMESIS_ENABLE_CKPOINT", "1",
+        HYDU_append_env_to_list("MPIR_CVAR_NEMESIS_ENABLE_CKPOINT", "1",
                                 &HYD_server_info.user_global.global_env.system);
 
     /* Preset common environment options for disabling STDIO buffering
diff --git a/src/util/mem/handlemem.c b/src/util/mem/handlemem.c
index 7894746..f565487 100644
--- a/src/util/mem/handlemem.c
+++ b/src/util/mem/handlemem.c
@@ -647,7 +647,7 @@ static int MPIU_CheckHandlesOnFinalize( void *objmem_ptr )
 	MPIU_Free( nIndirect );
     }
 
-    if (leaked_handles && MPIR_PARAM_ABORT_ON_LEAKED_HANDLES) {
+    if (leaked_handles && MPIR_CVAR_ABORT_ON_LEAKED_HANDLES) {
         /* comm_world has been (or should have been) destroyed by this point,
          * pass comm=NULL */
         MPID_Abort(NULL, MPI_ERR_OTHER, 1, "ERROR: leaked handles detected, aborting");
diff --git a/src/util/param/params.yml b/src/util/param/params.yml
index 9ea8d81..ee5576a 100644
--- a/src/util/param/params.yml
+++ b/src/util/param/params.yml
@@ -4,57 +4,57 @@
 # the inline list or map forms and don't try to use any really fancy
 # features like tagging or anchors.
 #
-# It is used to generate parameter-handling code for MPICH.
+# It is used to generate cvar-handling code for MPICH.
 # See the maint/genparams script for more information.
 #
 # The only tricky bit is that long reflowed text (such as a description)
 # should use ">-" to both fold newlines and remove the trailing newline.
 #
-# Listing "FOO" as a parameter name will cause two environment variables
-# to be examined by default: MPIR_PARAM_FOO and MPICH_FOO, with the
+# Listing "FOO" as a cvar name will cause two environment variables
+# to be examined by default: MPIR_CVAR_FOO and MPICH_FOO, with the
 # MPIR_PARAM value having priority.
 #
-# Each parameter has a class associated with it.  The following
+# Each cvar has a class associated with it.  The following
 # classes are defined: "device", and "none".  Setting it to "device"
 # will allow the device to override the default value.
 
 categories:
     - name        : COLLECTIVE
-      description : parameters that control collective communication behavior
+      description : cvars that control collective communication behavior
     - name        : COMMUNICATOR
-      description : parameters that control communicator construction and operation
+      description : cvars that control communicator construction and operation
     - name        : PT2PT
-      description : parameters that control point-to-point communication behavior
+      description : cvars that control point-to-point communication behavior
     - name        : RMA
-      description : parameters that control RMA communicatin behavior
+      description : cvars that control RMA communicatin behavior
     - name        : INTRANODE
-      description : intranode communication parameters
+      description : intranode communication cvars
     - name        : DEVELOPER
       description : useful for developers working on MPICH itself
     - name        : MEMORY
       description : affects memory allocation and usage, including MPI object handles
     - name        : ERROR_HANDLING
-      description : parameters that control error handling behavior (stack traces, aborts, etc)
+      description : cvars that control error handling behavior (stack traces, aborts, etc)
     - name        : DEBUGGER
-      description : parameters relevant to the "MPIR" debugger interface
+      description : cvars relevant to the "MPIR" debugger interface
     - name        : CHECKPOINTING
-      description : parameters relevant to checkpointing
+      description : cvars relevant to checkpointing
     - name        : FAULT_TOLERANCE
-      description : parameters that control fault tolerance behavior
+      description : cvars that control fault tolerance behavior
     - name        : THREADS
-      description : multi-threading parameters
+      description : multi-threading cvars
     - name        : CH3
-      description : parameters that control behavior of ch3
+      description : cvars that control behavior of ch3
     - name        : NEMESIS
-      description : "parameters that control behavior of the ch3:nemesis channel"
+      description : "cvars that control behavior of the ch3:nemesis channel"
     - name        : SOCKETS
-      description : control socket parameters
+      description : control socket cvars
     - name        : PROCESS_MANAGER
-      description : parameters that control the client-side process manager code
+      description : cvars that control the client-side process manager code
 
-parameters:
+cvars:
     ##############################################################
-    # collective parameters
+    # collective cvars
     - category    : COLLECTIVE
       name        : ALLTOALL_SHORT_MSG_SIZE
       type        : int
@@ -319,7 +319,7 @@ parameters:
         value of '0' uses SMP-aware allreduce for all message sizes.
 
     ##############################################################
-    # communicator parameters
+    # communicator cvars
     - category    : COMMUNICATOR
       name        : COMM_SPLIT_USE_QSORT
       type        : boolean
@@ -331,7 +331,7 @@ parameters:
         Use qsort(3) in the implementation of MPI_Comm_split instead of bubble sort.
 
   ##############################################################
-    # debugging parameters
+    # debugging cvars
     - category    : DEBUGGER
       name        : DEBUG_HOLD
       type        : boolean
@@ -368,7 +368,7 @@ parameters:
         (currently compile-time disabled by "#if 0")
 
   ##############################################################
-    # fault-tolerance parameters
+    # fault-tolerance cvars
     - category    : FAULT_TOLERANCE
       name        : ENABLE_COLL_FT_RET
       type        : boolean
@@ -385,7 +385,7 @@ parameters:
         if the result of the collective is invalid.
 
   ##############################################################
-    # memory parameters
+    # memory cvars
     - category    : MEMORY
       name        : ABORT_ON_LEAKED_HANDLES
       type        : boolean
@@ -403,7 +403,7 @@ parameters:
         work.
 
   ############################
-    # threads parameters
+    # threads cvars
     - category    : THREADS
       name        : CTXID_EAGER_SIZE
       type        : int
@@ -412,7 +412,7 @@ parameters:
       verbosity   : MPI_T_VERBOSITY_USER_BASIC
       scope       : MPI_T_SCOPE_LOCAL
       description : >-
-        The MPIR_PARAM_CTXID_EAGER_SIZE environment variable allows you to
+        The MPIR_CVAR_CTXID_EAGER_SIZE environment variable allows you to
         specify how many words in the context ID mask will be set aside
         for the eager allocation protocol.  If the application is running
         out of context IDs, reducing this value may help.
@@ -449,7 +449,7 @@ parameters:
         Sets the default thread level to use when using MPI_INIT.
 
     ##############################################################
-    # other MPI-level parameters
+    # other MPI-level cvars
     - category    : DEVELOPER
       name        : MEMDUMP
       type        : boolean
@@ -495,7 +495,7 @@ parameters:
         wide.  If 0, do not truncate, and if <0 use a sensible default.
 
     ##############################################################
-    # ch3 parameters
+    # ch3 cvars
     - category    : CH3
       name        : CH3_RMA_ACC_IMMED
       type        : boolean
@@ -592,10 +592,10 @@ parameters:
       verbosity   : MPI_T_VERBOSITY_USER_BASIC
       scope       : MPI_T_SCOPE_LOCAL
       description : >-
-        If non-NULL, this parameter specifies the IP address that
+        If non-NULL, this cvar specifies the IP address that
         other processes should use when connecting to this process.
-        This parameter is mutually exclusive with the
-        MPIR_PARAM_CH3_NETWORK_IFACE parameter and it is an error to set them
+        This cvar is mutually exclusive with the
+        MPIR_CVAR_CH3_NETWORK_IFACE cvar and it is an error to set them
         both.
 
     - category    : CH3
@@ -606,7 +606,7 @@ parameters:
       verbosity   : MPI_T_VERBOSITY_USER_BASIC
       scope       : MPI_T_SCOPE_LOCAL
       description : >-
-        This parameter controls the message size at which CH3 switches
+        This cvar controls the message size at which CH3 switches
         from eager to rendezvous mode.
 
     - category    : CH3
@@ -620,13 +620,13 @@ parameters:
       verbosity   : MPI_T_VERBOSITY_USER_BASIC
       scope       : MPI_T_SCOPE_LOCAL
       description : >-
-        The MPIR_PARAM_CH3_PORT_RANGE environment variable allows you to
+        The MPIR_CVAR_CH3_PORT_RANGE environment variable allows you to
         specify the range of TCP ports to be used by the process
         manager and the MPICH library. The format of this variable is
         <low>:<high>.  To specify any available port, use 0:0.
 
     ##############################################################
-    # nemesis parameters
+    # nemesis cvars
     - category    : NEMESIS
       name        : NEMESIS_POLLS_BEFORE_YIELD
       type        : int
@@ -636,7 +636,7 @@ parameters:
       scope       : MPI_T_SCOPE_LOCAL
       description : >-
         When MPICH is in a busy waiting loop, it will periodically
-        call a function to yield the processor.  This parameter sets
+        call a function to yield the processor.  This cvar sets
         the number of loops before the yield function is called.  A
         value of 0 disables yielding.
 
@@ -659,7 +659,7 @@ parameters:
       verbosity   : MPI_T_VERBOSITY_USER_BASIC
       scope       : MPI_T_SCOPE_LOCAL
       description : >-
-        If non-empty, this parameter specifies which network module
+        If non-empty, this cvar specifies which network module
         should be used for communication.
 
     - category    : NEMESIS
@@ -670,9 +670,9 @@ parameters:
       verbosity   : MPI_T_VERBOSITY_USER_BASIC
       scope       : MPI_T_SCOPE_LOCAL
       description : >-
-        This parameter controls the message size at which Nemesis
+        This cvar controls the message size at which Nemesis
         switches from eager to rendezvous mode for shared memory.
-        If this parameter is set to -1, then Nemesis will choose
+        If this cvar is set to -1, then Nemesis will choose
         an appropriate value.
 
     - category    : NEMESIS
@@ -683,10 +683,10 @@ parameters:
       verbosity   : MPI_T_VERBOSITY_USER_BASIC
       scope       : MPI_T_SCOPE_LOCAL
       description : >-
-        This parameter controls the message size at which Nemesis
+        This cvar controls the message size at which Nemesis
         switches from eager to rendezvous mode for ready-send
-        messages.  If this parameter is set to -1, then ready messages
-        will always be sent eagerly.  If this parameter is set to -2,
+        messages.  If this cvar is set to -1, then ready messages
+        will always be sent eagerly.  If this cvar is set to -2,
         then Nemesis will choose an appropriate value.
 
     - category    : NEMESIS
@@ -701,7 +701,7 @@ parameters:
         checkpointing library cannot be initialized.
 
     ##############################################################
-    # nemesis TCP parameters
+    # nemesis TCP cvars
     - category    : NEMESIS
       name        : NEMESIS_TCP_NETWORK_IFACE
       alt-env     :
@@ -712,11 +712,11 @@ parameters:
       verbosity   : MPI_T_VERBOSITY_USER_BASIC
       scope       : MPI_T_SCOPE_LOCAL
       description : >-
-        If non-NULL, this parameter specifies which pseudo-ethernet
+        If non-NULL, this cvar specifies which pseudo-ethernet
         interface the tcp netmod should use (e.g., "eth1", "ib0").
-        Note, this is a Linux-specific parameter.
-        This parameter is mutually exclusive with the
-        MPIR_PARAM_CH3_INTERFACE_HOSTNAME parameter and it is an error to set
+        Note, this is a Linux-specific cvar.
+        This cvar is mutually exclusive with the
+        MPIR_CVAR_CH3_INTERFACE_HOSTNAME cvar and it is an error to set
         them both.
 
     - category    : NEMESIS
@@ -727,11 +727,11 @@ parameters:
       verbosity   : MPI_T_VERBOSITY_USER_BASIC
       scope       : MPI_T_SCOPE_LOCAL
       description : >-
-        This parameter controls the number of times to retry the
+        This cvar controls the number of times to retry the
         gethostbyname() function before giving up.
 
     ##############################################################
-    # nemesis portals parameters
+    # nemesis portals cvars
     - category    : NEMESIS
       name        : NEMESIS_PORTALS_COMM_OVERRIDES
       type        : boolean
@@ -746,7 +746,7 @@ parameters:
         # setting the comm_ops table in any VC.
 
     ##############################################################
-    # process manager parameters
+    # process manager cvars
     - category    : PROCESS_MANAGER
       name        : NAMESERV_FILE_PUBDIR
       alt-env     :

http://git.mpich.org/mpich.git/commitdiff/fb570792e9bc6705bb7ede2286402544544d3561

commit fb570792e9bc6705bb7ede2286402544544d3561
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Fri Oct 18 19:49:54 2013 -0500

    Add a test case to test multithreading MPI_T
    
    The case is adapted from test/mpi/mpi_t/mpit_vars.c
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/test/mpi/.gitignore b/test/mpi/.gitignore
index 8155151..0a49c65 100644
--- a/test/mpi/.gitignore
+++ b/test/mpi/.gitignore
@@ -473,6 +473,7 @@
 /threads/pt2pt/threaded_sr
 /threads/pt2pt/alltoall
 /threads/pt2pt/sendselfth
+/threads/mpi_t/mpit_threading
 /threads/spawn/threaded_sr
 /threads/spawn/alltoall
 /threads/spawn/sendselfth
diff --git a/test/mpi/configure.ac b/test/mpi/configure.ac
index 220a3bd..ed5e4e7 100644
--- a/test/mpi/configure.ac
+++ b/test/mpi/configure.ac
@@ -1487,6 +1487,7 @@ AC_OUTPUT(maint/testmerge \
           threads/util/Makefile \
           threads/comm/Makefile \
           threads/init/Makefile \
+          threads/mpi_t/Makefile \
           threads/spawn/Makefile \
           threads/rma/Makefile \
           errors/Makefile \
diff --git a/test/mpi/threads/Makefile.am b/test/mpi/threads/Makefile.am
index bb3ec8c..421d45b 100644
--- a/test/mpi/threads/Makefile.am
+++ b/test/mpi/threads/Makefile.am
@@ -9,5 +9,5 @@ include $(top_srcdir)/Makefile.mtest
 
 EXTRA_DIST = testlist.in
 
-SUBDIRS = util pt2pt comm init $(spawndir) rma
-DIST_SUBDIRS = util pt2pt comm init spawn rma
\ No newline at end of file
+SUBDIRS = util pt2pt comm init mpi_t $(spawndir) rma
+DIST_SUBDIRS = util pt2pt comm init mpi_t spawn rma
diff --git a/test/mpi/threads/mpi_t/Makefile.am b/test/mpi/threads/mpi_t/Makefile.am
new file mode 100644
index 0000000..12fdd46
--- /dev/null
+++ b/test/mpi/threads/mpi_t/Makefile.am
@@ -0,0 +1,13 @@
+## -*- Mode: Makefile; -*-
+## vim: set ft=automake :
+##
+## (C) 2013 by Argonne National Laboratory.
+##     See COPYRIGHT in top-level directory.
+##
+
+include $(top_srcdir)/threads/Makefile_threads.mtest
+
+EXTRA_DIST = testlist
+
+noinst_PROGRAMS = mpit_threading
+
diff --git a/test/mpi/threads/mpi_t/mpit_threading.c b/test/mpi/threads/mpi_t/mpit_threading.c
new file mode 100644
index 0000000..9c08261
--- /dev/null
+++ b/test/mpi/threads/mpi_t/mpit_threading.c
@@ -0,0 +1,643 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
+/*
+ *  (C) 2013 by Argonne National Laboratory.
+ *      See COPYRIGHT in top-level directory.
+ */
+
+/* This test is adapted from test/mpi/mpi_t/mpit_vars.c.
+   But this is a multithreading version in which multiple threads will
+   call MPI_T routines.
+
+   If verbose is set, thread 0 will prints out MPI_T control variables,
+   performance variables and their categories.
+ */
+
+#include "mpi.h"
+#include "mpitest.h"
+#include "mpithreadtest.h"
+#include <stdio.h>
+#include <strings.h>
+#include <string.h>
+#include <stdlib.h>
+
+/* Num of threads to spawn */
+const int NTHREADS = 8;
+
+/* Use macro to control printing. Threads must define myThreadId.
+ */
+static int verbose = 0;
+#define DOPRINT (verbose && myThreadId == 0)
+
+char *mpit_scopeToStr(int scope);
+char *mpit_bindingToStr(int binding);
+char *mpit_validDtypeStr(MPI_Datatype datatype);
+char *mpit_varclassToStr(int varClass);
+char *mpit_verbosityToStr(int verbosity);
+int perfvarReadInt(int pvarIndex, int isContinuous, int *found);
+unsigned int perfvarReadUint(int pvarIndex, int isContinuous, int *found);
+double perfvarReadDouble(int pvarIndex, int isContinuous, int *found);
+int PrintControlVars(FILE * fp, int myThreadId);
+int PrintPerfVars(FILE * fp, int myThreadId);
+int PrintCategories(FILE * fp, int myThreadId);
+
+MTEST_THREAD_RETURN_TYPE RunTest(void *p)
+{
+    int myThreadId = (int)((intptr_t)p);
+
+    PrintControlVars(stdout, myThreadId);
+    if (DOPRINT)
+        fprintf(stdout, "\n");
+
+    PrintPerfVars(stdout, myThreadId);
+    if (DOPRINT)
+        fprintf(stdout, "\n");
+
+    PrintCategories(stdout, myThreadId);
+
+    return MTEST_THREAD_RETVAL_IGN;
+}
+
+int main(int argc, char *argv[])
+{
+    int i, required, provided;
+    required = MPI_THREAD_MULTIPLE;
+
+    MPI_T_init_thread(required, &provided);
+    if (provided != MPI_THREAD_MULTIPLE) {
+        fprintf(stderr, "MPI_THREAD_MULTIPLE not supported by the MPI_T implementation\n");
+        MPI_Abort(MPI_COMM_WORLD, -1);
+    }
+
+    MPI_Init_thread(&argc, &argv, required, &provided);
+    if (provided != MPI_THREAD_MULTIPLE) {
+        fprintf(stderr, "MPI_THREAD_MULTIPLE not supported by the MPI implementation\n");
+        MPI_Abort(MPI_COMM_WORLD, -1);
+    }
+
+    if (getenv("MPITEST_VERBOSE"))
+        verbose = 1;
+
+    /* Spawn threads */
+    for (i = 0; i < NTHREADS; i++) {
+        intptr_t tid = (intptr_t)i;
+        MTest_Start_thread(RunTest, (void *)tid);
+    }
+
+    MTest_Join_threads();
+
+    /* Put MPI_T_finalize() after MPI_Finalize() will cause mpich memory
+     * tracing facility falsely reports memory leaks, though these memories
+     * are freed in MPI_T_finalize().
+     */
+    MPI_T_finalize();
+    MPI_Finalize();
+
+    fprintf(stdout, " No Errors\n");
+
+    return 0;
+}
+
+int PrintControlVars(FILE * fp, int myThreadId)
+{
+    int i, num_cvar, nameLen, verbosity, descLen, binding, scope;
+    int ival, hasValue;
+    char name[128], desc[1024];
+    MPI_T_enum enumtype = MPI_T_ENUM_NULL;
+    MPI_Datatype datatype;
+
+    MPI_T_cvar_get_num(&num_cvar);
+    if (DOPRINT)
+        fprintf(fp, "%d MPI Control Variables\n", num_cvar);
+    for (i = 0; i < num_cvar; i++) {
+        hasValue = 0;
+        nameLen = sizeof(name);
+        descLen = sizeof(desc);
+        MPI_T_cvar_get_info(i, name, &nameLen, &verbosity, &datatype,
+                            &enumtype, desc, &descLen, &binding, &scope);
+        if (datatype == MPI_INT && enumtype != MPI_T_ENUM_NULL) {
+            int enameLen, enumber;
+            char ename[128];
+            enameLen = sizeof(ename);
+            /* TODO: Extract a useful string to show for an enum */
+            MPI_T_enum_get_info(enumtype, &enumber, ename, &enameLen);
+        }
+        if (datatype == MPI_INT && binding == MPI_T_BIND_NO_OBJECT) {
+            int count;
+            MPI_T_cvar_handle chandle;
+            MPI_T_cvar_handle_alloc(i, NULL, &chandle, &count);
+            if (count == 1) {
+                MPI_T_cvar_read(chandle, &ival);
+                hasValue = 1;
+            }
+            MPI_T_cvar_handle_free(&chandle);
+        }
+
+        if (hasValue && DOPRINT) {
+            fprintf(fp, "\t%s=%d\t%s\t%s\t%s\t%s\t%s\n",
+                    name,
+                    ival,
+                    mpit_scopeToStr(scope),
+                    mpit_bindingToStr(binding),
+                    mpit_validDtypeStr(datatype), mpit_verbosityToStr(verbosity), desc);
+        }
+        else if (DOPRINT) {
+            fprintf(fp, "\t%s\t%s\t%s\t%s\t%s\t%s\n",
+                    name,
+                    mpit_scopeToStr(scope),
+                    mpit_bindingToStr(binding),
+                    mpit_validDtypeStr(datatype), mpit_verbosityToStr(verbosity), desc);
+        }
+    }
+
+    return 0;
+}
+
+int PrintPerfVars(FILE * fp, int myThreadId)
+{
+    int i, numPvar, nameLen, descLen, verbosity, varClass;
+    int binding, isReadonly, isContinuous, isAtomic;
+    char name[128], desc[1024];
+    MPI_T_enum enumtype;
+    MPI_Datatype datatype;
+
+    MPI_T_pvar_get_num(&numPvar);
+    if (DOPRINT)
+        fprintf(fp, "%d MPI Performance Variables\n", numPvar);
+
+    for (i = 0; i < numPvar; i++) {
+        nameLen = sizeof(name);
+        descLen = sizeof(desc);
+        MPI_T_pvar_get_info(i, name, &nameLen, &verbosity, &varClass,
+                            &datatype, &enumtype, desc, &descLen, &binding,
+                            &isReadonly, &isContinuous, &isAtomic);
+
+        if (DOPRINT)
+            fprintf(fp, "\t%s\t%s\t%s\t%s\t%s\tReadonly=%s\tContinuous=%s\tAtomic=%s\t%s\n",
+                    name,
+                    mpit_varclassToStr(varClass),
+                    mpit_bindingToStr(binding),
+                    mpit_validDtypeStr(datatype),
+                    mpit_verbosityToStr(verbosity),
+                    isReadonly ? "T" : "F", isContinuous ? "T" : "F", isAtomic ? "T" : "F", desc);
+
+        if (datatype == MPI_INT) {
+            int val, isFound;
+            val = perfvarReadInt(i, isContinuous, &isFound);
+            if (isFound && DOPRINT)
+                fprintf(fp, "\tValue = %d\n", val);
+        }
+        else if (datatype == MPI_UNSIGNED) {
+            int isFound;
+            unsigned int val;
+            val = perfvarReadUint(i, isContinuous, &isFound);
+            if (isFound && DOPRINT)
+                fprintf(fp, "\tValue = %u\n", val);
+        }
+        else if (datatype == MPI_DOUBLE) {
+            int isFound;
+            double val;
+            val = perfvarReadDouble(i, isContinuous, &isFound);
+            if (isFound && DOPRINT)
+                fprintf(fp, "\tValue = %e\n", val);
+        }
+    }
+    return 0;
+}
+
+int PrintCategories(FILE * fp, int myThreadId)
+{
+    int i, j, numCat, nameLen, descLen, numCvars, numPvars, numSubcat;
+    char name[128], desc[1024];
+
+    MPI_T_category_get_num(&numCat);
+    if (DOPRINT) {
+        if (numCat > 0)
+            fprintf(fp, "%d MPI_T categories\n", numCat);
+        else
+            fprintf(fp, "No categories defined\n");
+    }
+
+    for (i = 0; i < numCat; i++) {
+        nameLen = sizeof(name);
+        descLen = sizeof(desc);
+        MPI_T_category_get_info(i, name, &nameLen, desc, &descLen, &numCvars,
+                                &numPvars, &numSubcat);
+        if (DOPRINT)
+            fprintf(fp, "Category %s has %d control variables, %d performance variables, %d subcategories\n",
+                    name, numCvars, numPvars, numSubcat);
+
+        if (numCvars > 0) {
+            if (DOPRINT)
+                fprintf(fp, "\tControl variables include: ");
+            int *cvarIndex = (int *) malloc(numCvars * sizeof(int));
+            MPI_T_category_get_cvars(i, numCvars, cvarIndex);
+            for (j = 0; j < numCvars; j++) {
+                /* Get just the variable name */
+                int varnameLen, verb, binding, scope;
+                MPI_Datatype datatype;
+                char varname[128];
+                varnameLen = sizeof(varname);
+                MPI_T_cvar_get_info(cvarIndex[j], varname, &varnameLen,
+                                    &verb, &datatype, NULL, NULL, NULL, &binding, &scope);
+                if (DOPRINT)
+                    fprintf(fp, "%s, ", varname);
+            }
+            free(cvarIndex);
+            if (DOPRINT)
+                fprintf(fp, "\n");
+        }
+
+        if (numPvars > 0) {
+            if (DOPRINT)
+                fprintf(fp, "\tPerformance variables include: ");
+
+            int *pvarIndex = (int *) malloc(numPvars * sizeof(int));
+            MPI_T_category_get_pvars(i, numPvars, pvarIndex);
+            for (j = 0; j < numPvars; j++) {
+                int varnameLen, verb, varclass, binding;
+                int isReadonly, isContinuous, isAtomic;
+                MPI_Datatype datatype;
+                char varname[128];
+                varnameLen = sizeof(varname);
+                MPI_T_pvar_get_info(pvarIndex[j], varname, &varnameLen, &verb,
+                                    &varclass, &datatype, NULL, NULL, NULL, &binding,
+                                    &isReadonly, &isContinuous, &isAtomic);
+                if (DOPRINT)
+                    fprintf(fp, "%s, ", varname);
+
+            }
+            free(pvarIndex);
+            if (DOPRINT)
+                fprintf(fp, "\n");
+        }
+
+        if (numSubcat > 0) {
+            if (DOPRINT)
+                fprintf(fp, "\tSubcategories include: ");
+
+            int *subcatIndex = (int *) malloc(numSubcat * sizeof(int));
+            MPI_T_category_get_categories(i, numSubcat, subcatIndex);
+            for (j = 0; j < numSubcat; j++) {
+                int catnameLen, ncvars, npvars, nsubcats;
+                char catname[128];
+                catnameLen = sizeof(catname);
+                MPI_T_category_get_info(subcatIndex[j], catname, &catnameLen, NULL, NULL,
+                                        &ncvars, &npvars, &nsubcats);
+                if (DOPRINT)
+                    fprintf(fp, "%s, ", catname);
+
+            }
+            free(subcatIndex);
+            if (DOPRINT)
+                fprintf(fp, "\n");
+        }
+    }
+
+    return 0;
+}
+
+
+/* --- Support routines --- */
+
+char *mpit_validDtypeStr(MPI_Datatype datatype)
+{
+    char *p = 0;
+    if (datatype == MPI_INT)
+        p = "MPI_INT";
+    else if (datatype == MPI_UNSIGNED)
+        p = "MPI_UNSIGNED";
+    else if (datatype == MPI_UNSIGNED_LONG)
+        p = "MPI_UNSIGNED_LONG";
+    else if (datatype == MPI_UNSIGNED_LONG_LONG)
+        p = "MPI_UNSIGNED_LONG_LONG";
+    else if (datatype == MPI_COUNT)
+        p = "MPI_COUNT";
+    else if (datatype == MPI_CHAR)
+        p = "MPI_CHAR";
+    else if (datatype == MPI_DOUBLE)
+        p = "MPI_DOUBLE";
+    else {
+        if (datatype == MPI_DATATYPE_NULL) {
+            p = "Invalid MPI datatype:NULL";
+        }
+        else {
+            static char typename[MPI_MAX_OBJECT_NAME + 9];
+            int tlen;
+            strncpy(typename, "Invalid:", MPI_MAX_OBJECT_NAME);
+            MPI_Type_get_name(datatype, typename + 8, &tlen);
+            if (typename[0])
+                p = typename;
+        }
+    }
+
+    return p;
+}
+
+char *mpit_scopeToStr(int scope)
+{
+    char *p = 0;
+    switch (scope) {
+    case MPI_T_SCOPE_CONSTANT:
+        p = "SCOPE_CONSTANT";
+        break;
+    case MPI_T_SCOPE_READONLY:
+        p = "SCOPE_READONLY";
+        break;
+    case MPI_T_SCOPE_LOCAL:
+        p = "SCOPE_LOCAL";
+        break;
+    case MPI_T_SCOPE_GROUP:
+        p = "SCOPE_GROUP";
+        break;
+    case MPI_T_SCOPE_GROUP_EQ:
+        p = "SCOPE_GROUP_EQ";
+        break;
+    case MPI_T_SCOPE_ALL:
+        p = "SCOPE_ALL";
+        break;
+    case MPI_T_SCOPE_ALL_EQ:
+        p = "SCOPE_ALL_EQ";
+        break;
+    default:
+        p = "Unrecoginized scope";
+        break;
+    }
+    return p;
+}
+
+char *mpit_bindingToStr(int binding)
+{
+    char *p;
+    switch (binding) {
+    case MPI_T_BIND_NO_OBJECT:
+        p = "NO_OBJECT";
+        break;
+    case MPI_T_BIND_MPI_COMM:
+        p = "MPI_COMM";
+        break;
+    case MPI_T_BIND_MPI_DATATYPE:
+        p = "MPI_DATATYPE";
+        break;
+    case MPI_T_BIND_MPI_ERRHANDLER:
+        p = "MPI_ERRHANDLER";
+        break;
+    case MPI_T_BIND_MPI_FILE:
+        p = "MPI_FILE";
+        break;
+    case MPI_T_BIND_MPI_GROUP:
+        p = "MPI_GROUP";
+        break;
+    case MPI_T_BIND_MPI_OP:
+        p = "MPI_OP";
+        break;
+    case MPI_T_BIND_MPI_REQUEST:
+        p = "MPI_REQUEST";
+        break;
+    case MPI_T_BIND_MPI_WIN:
+        p = "MPI_WIN";
+        break;
+    case MPI_T_BIND_MPI_MESSAGE:
+        p = "MPI_MESSAGE";
+        break;
+    case MPI_T_BIND_MPI_INFO:
+        p = "MPI_INFO";
+        break;
+    default:
+        p = "Unknown object binding";
+    }
+    return p;
+}
+
+char *mpit_varclassToStr(int varClass)
+{
+    char *p = 0;
+    switch (varClass) {
+    case MPI_T_PVAR_CLASS_STATE:
+        p = "CLASS_STATE";
+        break;
+    case MPI_T_PVAR_CLASS_LEVEL:
+        p = "CLASS_LEVEL";
+        break;
+    case MPI_T_PVAR_CLASS_SIZE:
+        p = "CLASS_SIZE";
+        break;
+    case MPI_T_PVAR_CLASS_PERCENTAGE:
+        p = "CLASS_PERCENTAGE";
+        break;
+    case MPI_T_PVAR_CLASS_HIGHWATERMARK:
+        p = "CLASS_HIGHWATERMARK";
+        break;
+    case MPI_T_PVAR_CLASS_LOWWATERMARK:
+        p = "CLASS_LOWWATERMARK";
+        break;
+    case MPI_T_PVAR_CLASS_COUNTER:
+        p = "CLASS_COUNTER";
+        break;
+    case MPI_T_PVAR_CLASS_AGGREGATE:
+        p = "CLASS_AGGREGATE";
+        break;
+    case MPI_T_PVAR_CLASS_TIMER:
+        p = "CLASS_TIMER";
+        break;
+    case MPI_T_PVAR_CLASS_GENERIC:
+        p = "CLASS_GENERIC";
+        break;
+    default:
+        p = "Unrecognized pvar class";
+        break;
+    }
+    return p;
+}
+
+char *mpit_verbosityToStr(int verbosity)
+{
+    char *p = 0;
+    switch (verbosity) {
+    case MPI_T_VERBOSITY_USER_BASIC:
+        p = "VERBOSITY_USER_BASIC";
+        break;
+    case MPI_T_VERBOSITY_USER_DETAIL:
+        p = "VERBOSITY_USER_DETAIL";
+        break;
+    case MPI_T_VERBOSITY_USER_ALL:
+        p = "VERBOSITY_USER_ALL";
+        break;
+    case MPI_T_VERBOSITY_TUNER_BASIC:
+        p = "VERBOSITY_TUNER_BASIC";
+        break;
+    case MPI_T_VERBOSITY_TUNER_DETAIL:
+        p = "VERBOSITY_TUNER_DETAIL";
+        break;
+    case MPI_T_VERBOSITY_TUNER_ALL:
+        p = "VERBOSITY_TUNER_ALL";
+        break;
+    case MPI_T_VERBOSITY_MPIDEV_BASIC:
+        p = "VERBOSITY_MPIDEV_BASIC";
+        break;
+    case MPI_T_VERBOSITY_MPIDEV_DETAIL:
+        p = "VERBOSITY_MPIDEV_DETAIL";
+        break;
+    case MPI_T_VERBOSITY_MPIDEV_ALL:
+        p = "VERBOSITY_MPIDEV_ALL";
+        break;
+    default:
+        p = "Invalid verbosity";
+        break;
+    }
+    return p;
+}
+
+char *mpit_errclassToStr(int err)
+{
+    char *p = 0;
+    switch (err) {
+    case MPI_T_ERR_MEMORY:
+        p = "ERR_MEMORY";
+        break;
+    case MPI_T_ERR_NOT_INITIALIZED:
+        p = "ERR_NOT_INITIALIZED";
+        break;
+    case MPI_T_ERR_CANNOT_INIT:
+        p = "ERR_CANNOT_INIT";
+        break;
+    case MPI_T_ERR_INVALID_INDEX:
+        p = "ERR_INVALID_INDEX";
+        break;
+    case MPI_T_ERR_INVALID_ITEM:
+        p = "ERR_INVALID_ITEM";
+        break;
+    case MPI_T_ERR_INVALID_HANDLE:
+        p = "ERR_INVALID_HANDLE";
+        break;
+    case MPI_T_ERR_OUT_OF_HANDLES:
+        p = "ERR_OUT_OF_HANDLES";
+        break;
+    case MPI_T_ERR_OUT_OF_SESSIONS:
+        p = "ERR_OUT_OF_SESSIONS";
+        break;
+    case MPI_T_ERR_INVALID_SESSION:
+        p = "ERR_INVALID_SESSION";
+        break;
+    case MPI_T_ERR_CVAR_SET_NOT_NOW:
+        p = "ERR_CVAR_SET_NOT_NOW";
+        break;
+    case MPI_T_ERR_CVAR_SET_NEVER:
+        p = "ERR_CVAR_SET_NEVER";
+        break;
+    case MPI_T_ERR_PVAR_NO_STARTSTOP:
+        p = "ERR_PVAR_NO_STARTSTOP";
+        break;
+    case MPI_T_ERR_PVAR_NO_WRITE:
+        p = "ERR_PVAR_NO_WRITE";
+        break;
+    case MPI_T_ERR_PVAR_NO_ATOMIC:
+        p = "ERR_PVAR_NO_ATOMIC";
+        break;
+    default:
+        p = "Unknown MPI_T_ERR class";
+        break;
+    }
+    return p;
+}
+
+/* Return the value of the performance variable as the value */
+int perfvarReadInt(int pvarIndex, int isContinuous, int *found)
+{
+    int count, val = -1;
+    int err1 = MPI_SUCCESS;
+    int err2 = MPI_SUCCESS;
+    MPI_T_pvar_session session;
+    MPI_T_pvar_handle pvarHandle;
+    MPI_T_pvar_session_create(&session);
+    MPI_T_pvar_handle_alloc(session, pvarIndex, NULL, &pvarHandle, &count);
+    if (count == 1) {
+        *found = 1;
+        if (!isContinuous) {
+            /* start and stop the variable (just because we can) */
+            err1 = MPI_T_pvar_start(session, pvarHandle);
+            err2 = MPI_T_pvar_stop(session, pvarHandle);
+        }
+        MPI_T_pvar_read(session, pvarHandle, &val);
+    }
+    MPI_T_pvar_handle_free(session, &pvarHandle);
+    MPI_T_pvar_session_free(&session);
+
+    /* Above codes imply that err1 and err2 should be MPI_SUCCESS.
+     * If not, catch errors here, e.g., when MPI_ERR_INTERN is returned.
+     */
+    if (err1 != MPI_SUCCESS || err2 != MPI_SUCCESS) {
+        fprintf(stderr, "Unexpected MPI_T_pvar_start/stop return code\n");
+        abort();
+    }
+
+    return val;
+}
+
+/* Return the value of the performance variable as the value */
+unsigned int perfvarReadUint(int pvarIndex, int isContinuous, int *found)
+{
+    int count;
+    unsigned int val = 0;
+    int err1 = MPI_SUCCESS;
+    int err2 = MPI_SUCCESS;
+    MPI_T_pvar_session session;
+    MPI_T_pvar_handle pvarHandle;
+
+    *found = 0;
+    MPI_T_pvar_session_create(&session);
+    MPI_T_pvar_handle_alloc(session, pvarIndex, NULL, &pvarHandle, &count);
+    if (count == 1) {
+        *found = 1;
+        if (!isContinuous) {
+            /* start and stop the variable (just because we can) */
+            err1 = MPI_T_pvar_start(session, pvarHandle);
+            err2 = MPI_T_pvar_stop(session, pvarHandle);
+        }
+        MPI_T_pvar_read(session, pvarHandle, &val);
+    }
+    MPI_T_pvar_handle_free(session, &pvarHandle);
+    MPI_T_pvar_session_free(&session);
+
+    /* Above codes imply that err1 and err2 should be MPI_SUCCESS.
+     * If not, catch errors here, e.g., when MPI_ERR_INTERN is returned.
+     */
+    if (err1 != MPI_SUCCESS || err2 != MPI_SUCCESS) {
+        fprintf(stderr, "Unexpected MPI_T_pvar_start/stop return code\n");
+        abort();
+    }
+
+    return val;
+}
+
+double perfvarReadDouble(int pvarIndex, int isContinuous, int *found)
+{
+    int count;
+    double val = 0.0;
+    int err1 = MPI_SUCCESS;
+    int err2 = MPI_SUCCESS;
+    MPI_T_pvar_session session;
+    MPI_T_pvar_handle pvarHandle;
+
+    *found = 0;
+    MPI_T_pvar_session_create(&session);
+    MPI_T_pvar_handle_alloc(session, pvarIndex, NULL, &pvarHandle, &count);
+    if (count == 1) {
+        *found = 1;
+        if (!isContinuous) {
+            /* start and stop the variable (just because we can) */
+            err1 = MPI_T_pvar_start(session, pvarHandle);
+            err2 = MPI_T_pvar_stop(session, pvarHandle);
+        }
+        MPI_T_pvar_read(session, pvarHandle, &val);
+    }
+    MPI_T_pvar_handle_free(session, &pvarHandle);
+    MPI_T_pvar_session_free(&session);
+
+    /* Catch errors if MPI_T_pvar_start/stop are not properly implemented */
+    if (err1 != MPI_SUCCESS || err2 != MPI_SUCCESS) {
+        fprintf(stderr, "Unknown MPI_T return code when starting/stopping double pvar\n");
+        abort();
+    }
+
+    return val;
+}
diff --git a/test/mpi/threads/mpi_t/testlist b/test/mpi/threads/mpi_t/testlist
new file mode 100644
index 0000000..83515c2
--- /dev/null
+++ b/test/mpi/threads/mpi_t/testlist
@@ -0,0 +1 @@
+mpit_threading 1
diff --git a/test/mpi/threads/testlist.in b/test/mpi/threads/testlist.in
index 934ff26..6854dd7 100644
--- a/test/mpi/threads/testlist.in
+++ b/test/mpi/threads/testlist.in
@@ -1,5 +1,6 @@
 pt2pt
 comm
 init
+mpi_t
 @spawndir@
 rma

http://git.mpich.org/mpich.git/commitdiff/de70f872aa9b3c64c67aee1beeb5eadef2830b60

commit de70f872aa9b3c64c67aee1beeb5eadef2830b60
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Thu Oct 17 12:35:28 2013 -0500

    Delete xfail=ticket1898 of mpi_t_str
    
    This bug was fixed with the new MPI_T implementation.
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/test/mpi/mpi_t/testlist b/test/mpi/mpi_t/testlist
index effe91f..69fcab4 100644
--- a/test/mpi/mpi_t/testlist
+++ b/test/mpi/mpi_t/testlist
@@ -1,2 +1,2 @@
-mpi_t_str 1 mpiversion=3.0 xfail=ticket1898
+mpi_t_str 1 mpiversion=3.0
 mpit_vars 1 mpiversion=3.0

http://git.mpich.org/mpich.git/commitdiff/9fcd57127c9dfa50e7dd06162dbed9f131139de3

commit 9fcd57127c9dfa50e7dd06162dbed9f131139de3
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Fri Oct 18 13:58:44 2013 -0500

    Revise two tests under test/mpi/manual/mpi_t/
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/test/mpi/manual/mpi_t/nem_fbox_fallback_to_queue_count.c b/test/mpi/manual/mpi_t/nem_fbox_fallback_to_queue_count.c
index 04c1cba..640d5c1 100644
--- a/test/mpi/manual/mpi_t/nem_fbox_fallback_to_queue_count.c
+++ b/test/mpi/manual/mpi_t/nem_fbox_fallback_to_queue_count.c
@@ -34,7 +34,9 @@ int err, rank;
 MPI_T_pvar_session session;
 MPI_T_pvar_handle fbox_handle;
 
-/* Check that we can successfuly write to the variable. */
+/* Check that we can successfuly write to the variable.
+ * Question: Do we really want to write pvars other than reset?
+ */
 void blank_test()
 {
     uint64_t temp[2] = { -1 };
@@ -57,7 +59,7 @@ void send_first_test()
     uint64_t nem_fbox_fall_back_to_queue_count[2] = {-1};
 
     /* Reset the fbox variable. */
-    MPI_T_pvar_write(session, fbox_handle, null_fbox);
+    MPI_T_pvar_reset(session, fbox_handle);
 
     if (rank == 0) {
         char send_buf[BUF_COUNT] = { 0x12 };
@@ -117,7 +119,7 @@ void recv_first_test()
     uint64_t nem_fbox_fall_back_to_queue_count[2] = {-1};
 
     /* Reset the fbox variable. */
-    MPI_T_pvar_write(session, fbox_handle, null_fbox);
+    MPI_T_pvar_reset(session, fbox_handle);
 
     if (rank == 0) {
         char send_buf[BUF_COUNT] = { 0x12 };
@@ -211,7 +213,7 @@ int main(int argc, char *argv[])
     assert(count == 2);
 
     /* Run a batch of tests. */
-    blank_test();
+    /* blank_test(); */
     send_first_test();
     /* recv_first_test(); */
 
diff --git a/test/mpi/manual/mpi_t/unexpected_recvq_buffer_size.c b/test/mpi/manual/mpi_t/unexpected_recvq_buffer_size.c
index 4753e02..a0286db 100644
--- a/test/mpi/manual/mpi_t/unexpected_recvq_buffer_size.c
+++ b/test/mpi/manual/mpi_t/unexpected_recvq_buffer_size.c
@@ -120,6 +120,8 @@ int main(int argc, char *argv[])
     TRY(MPI_T_init_thread(MPI_THREAD_SINGLE, &thread_support));
     TRY(MPI_T_pvar_get_num(&num));
 
+    int found = 0;
+
     /* Locate desired MPIT variable. */
     for (i = 0; i < num; i++) {
         name_len = desc_len = STR_LEN;
@@ -127,24 +129,28 @@ int main(int argc, char *argv[])
                                 &enumtype, desc, &desc_len, &bind, &readonly,
                                 &continuous, &atomic));
 
-        if (strcmp(name, "unexpected_recvq_buffer_size") == 0)
+        if (strcmp(name, "unexpected_recvq_buffer_size") == 0) {
             uqsize_idx = i;
+            found = 1;
+        }
     }
 
-    /* Initialize MPIT session & variable handle. */
-    MPI_T_pvar_session_create(&session);
-    MPI_T_pvar_handle_alloc(session, uqsize_idx, NULL, &uqsize_handle, &count);
+    if (found) {
+        /* Initialize MPIT session & variable handle. */
+        MPI_T_pvar_session_create(&session);
+        MPI_T_pvar_handle_alloc(session, uqsize_idx, NULL, &uqsize_handle, &count);
 
-    /* Ensure the variable is of the correct size. */
-    assert(count == 1);
+        /* Ensure the variable is of the correct size. */
+        assert(count == 1);
 
-    /* Run a batch of tests. */
-    reversed_tags_test();
-    rndv_test();
+        /* Run a batch of tests. */
+        reversed_tags_test();
+        rndv_test();
 
-    /* Cleanup. */
-    MPI_T_pvar_handle_free(session, &uqsize_handle);
-    MPI_T_pvar_session_free(&session);
+        /* Cleanup. */
+        MPI_T_pvar_handle_free(session, &uqsize_handle);
+        MPI_T_pvar_session_free(&session);
+    }
 
     if (rank == 0) {
         printf("finished\n");

http://git.mpich.org/mpich.git/commitdiff/d5ca1acdb06e69de322fb87e2c13094ef35c2cc9

commit d5ca1acdb06e69de322fb87e2c13094ef35c2cc9
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Fri Oct 18 12:30:49 2013 -0500

    Revise the test case mpit_vars.c
    
    Now we can control printing using env var MPITEST_VERBOSE.
    The old code used a local variable 'verbose'. It shades the global
    variable 'verbose'. So we renamed the local one to 'verb' instead.
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/test/mpi/mpi_t/mpit_vars.c b/test/mpi/mpi_t/mpit_vars.c
index 3798764..6954466 100644
--- a/test/mpi/mpi_t/mpit_vars.c
+++ b/test/mpi/mpi_t/mpit_vars.c
@@ -37,6 +37,9 @@ int main(int argc, char *argv[])
     MPI_T_init_thread(required, &provided);
     MPI_Init_thread(&argc, &argv, required, &provided);
 
+    if (getenv("MPITEST_VERBOSE"))
+        verbose = 1;
+
     PrintControlVars(stdout);
     if (verbose)
         fprintf(stdout, "\n");
@@ -47,8 +50,12 @@ int main(int argc, char *argv[])
 
     PrintCategories(stdout);
 
-    MPI_Finalize();
+    /* Put MPI_T_finalize() after MPI_Finalize() will cause mpich memory
+     * tracing facility falsely reports memory leaks, though these memories
+     * are freed in MPI_T_finalize().
+     */
     MPI_T_finalize();
+    MPI_Finalize();
 
     fprintf(stdout, " No Errors\n");
 
@@ -176,11 +183,14 @@ int PrintCategories(FILE * fp)
     }
 
     for (i = 0; i < numCat; i++) {
+        nameLen = sizeof(name);
+        descLen = sizeof(desc);
         MPI_T_category_get_info(i, name, &nameLen, desc, &descLen, &numCvars,
                                 &numPvars, &numSubcat);
         if (verbose)
-            fprintf(fp, "Category %s has %d control variables, %d performance variables, \
-                %d subcategories\n", name, numCvars, numPvars, numSubcat);
+            fprintf(fp, "Category %s has %d control variables, %d performance variables, %d subcategories\n",
+                    name, numCvars, numPvars, numSubcat);
+            fprintf(fp, "\tDescription: %s\n", desc);
 
         if (numCvars > 0) {
             if (verbose)
@@ -189,12 +199,12 @@ int PrintCategories(FILE * fp)
             MPI_T_category_get_cvars(i, numCvars, cvarIndex);
             for (j = 0; j < numCvars; j++) {
                 /* Get just the variable name */
-                int varnameLen, verbose, binding, scope;
+                int varnameLen, verb, binding, scope;
                 MPI_Datatype datatype;
                 char varname[128];
                 varnameLen = sizeof(varname);
                 MPI_T_cvar_get_info(cvarIndex[j], varname, &varnameLen,
-                                    &verbose, &datatype, NULL, NULL, NULL, &binding, &scope);
+                                    &verb, &datatype, NULL, NULL, NULL, &binding, &scope);
                 if (verbose)
                     fprintf(fp, "%s, ", varname);
             }
@@ -210,12 +220,12 @@ int PrintCategories(FILE * fp)
             int *pvarIndex = (int *) malloc(numPvars * sizeof(int));
             MPI_T_category_get_pvars(i, numPvars, pvarIndex);
             for (j = 0; j < numPvars; j++) {
-                int varnameLen, verbose, varclass, binding;
+                int varnameLen, verb, varclass, binding;
                 int isReadonly, isContinuous, isAtomic;
                 MPI_Datatype datatype;
                 char varname[128];
                 varnameLen = sizeof(varname);
-                MPI_T_pvar_get_info(pvarIndex[j], varname, &varnameLen, &verbose,
+                MPI_T_pvar_get_info(pvarIndex[j], varname, &varnameLen, &verb,
                                     &varclass, &datatype, NULL, NULL, NULL, &binding,
                                     &isReadonly, &isContinuous, &isAtomic);
                 if (verbose)
@@ -228,8 +238,26 @@ int PrintCategories(FILE * fp)
         }
 
         /* TODO: Make it possible to recursively print category information */
-        if (verbose)
-            fprintf(fp, "\tNumber of subcategories: %d\n", numSubcat);
+        if (numSubcat > 0) {
+            if (verbose)
+                fprintf(fp, "\tSubcategories include: ");
+
+            int *subcatIndex = (int *) malloc(numSubcat * sizeof(int));
+            MPI_T_category_get_categories(i, numSubcat, subcatIndex);
+            for (j = 0; j < numSubcat; j++) {
+                int catnameLen, ncvars, npvars, nsubcats;
+                char catname[128];
+                catnameLen = sizeof(catname);
+                MPI_T_category_get_info(subcatIndex[j], catname, &catnameLen, NULL, NULL,
+                                        &ncvars, &npvars, &nsubcats);
+                if (verbose)
+                    fprintf(fp, "%s, ", catname);
+
+            }
+            free(subcatIndex);
+            if (verbose)
+                fprintf(fp, "\n");
+        }
     }
 
     return 0;

http://git.mpich.org/mpich.git/commitdiff/83f6c94a86abb4b8875e6ecc88e24d8a7ac0e8e4

commit 83f6c94a86abb4b8875e6ecc88e24d8a7ac0e8e4
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Fri Oct 18 11:29:36 2013 -0500

    Rename mpit_init_thread.c to mpit_initthread.c
    
    Since initthread.c is used for MPI_Init_thread(), we'd better use
    mpit_initthread.c for MPI_T_init_thread().
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/src/mpi_t/Makefile.mk b/src/mpi_t/Makefile.mk
index 8402e0e..37e69da 100644
--- a/src/mpi_t/Makefile.mk
+++ b/src/mpi_t/Makefile.mk
@@ -21,7 +21,7 @@ mpi_sources +=                \
         src/mpi_t/enum_get_info.c       \
         src/mpi_t/enum_get_item.c       \
         src/mpi_t/mpit_finalize.c       \
-        src/mpi_t/mpit_init_thread.c    \
+        src/mpi_t/mpit_initthread.c     \
         src/mpi_t/pvar_get_info.c       \
         src/mpi_t/pvar_get_num.c        \
         src/mpi_t/pvar_handle_alloc.c   \
diff --git a/src/mpi_t/mpit_init_thread.c b/src/mpi_t/mpit_initthread.c
similarity index 100%
rename from src/mpi_t/mpit_init_thread.c
rename to src/mpi_t/mpit_initthread.c

http://git.mpich.org/mpich.git/commitdiff/601ac99d425e8d0dff6f584aac41e4878d2b405a

commit 601ac99d425e8d0dff6f584aac41e4878d2b405a
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Fri Oct 18 11:25:02 2013 -0500

    Rename cat_get_pv.c to cat_get_pvars.c
    
    The old name doesn't confrom to naming convention of other files under mpi_t
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/src/mpi_t/Makefile.mk b/src/mpi_t/Makefile.mk
index 98d5404..8402e0e 100644
--- a/src/mpi_t/Makefile.mk
+++ b/src/mpi_t/Makefile.mk
@@ -11,7 +11,7 @@ mpi_sources +=                \
         src/mpi_t/cat_get_cvars.c       \
         src/mpi_t/cat_get_info.c        \
         src/mpi_t/cat_get_num.c         \
-        src/mpi_t/cat_get_pv.c          \
+        src/mpi_t/cat_get_pvars.c       \
         src/mpi_t/cvar_get_info.c       \
         src/mpi_t/cvar_get_num.c        \
         src/mpi_t/cvar_handle_alloc.c   \
diff --git a/src/mpi_t/cat_get_pv.c b/src/mpi_t/cat_get_pvars.c
similarity index 100%
rename from src/mpi_t/cat_get_pv.c
rename to src/mpi_t/cat_get_pvars.c

http://git.mpich.org/mpich.git/commitdiff/fc51e3ad893e6ab11feec5b4f6edfd85d1773f02

commit fc51e3ad893e6ab11feec5b4f6edfd85d1773f02
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Fri Oct 18 09:50:26 2013 -0500

    Rename mpich_param_vals.h, param_vals.c
    
    These two files are renamed to mpich_params.{c, h} since the old
    ones are not uniform.
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/.gitignore b/.gitignore
index ed5203c..9a0a2f0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -224,8 +224,8 @@ Makefile.am-stamp
 /src/binding/f90/Makefile.mk
 
 # MPICH2 parameter handling
-/src/include/mpich_param_vals.h
-/src/util/param/param_vals.c
+/src/include/mpich_params.h
+/src/util/param/mpich_params.c
 
 
 ################################################################################
diff --git a/maint/genparams b/maint/genparams
index 5e7217b..dad5841 100755
--- a/maint/genparams
+++ b/maint/genparams
@@ -44,8 +44,8 @@ my $alt_ns = "MPICH";
 my $param_file = "src/util/param/params.yml";
 
 # output source files
-my $header_file = "src/include/mpich_param_vals.h";
-my $c_file      = "src/util/param/param_vals.c";
+my $header_file = "src/include/mpich_params.h";
+my $c_file      = "src/util/param/mpich_params.c";
 my $readme_file = "README.envvar";
 
 GetOptions(
diff --git a/src/include/Makefile.mk b/src/include/Makefile.mk
index 4165748..fab495e 100644
--- a/src/include/Makefile.mk
+++ b/src/include/Makefile.mk
@@ -34,7 +34,7 @@ noinst_HEADERS +=                   \
     src/include/mpiallstates.h      \
     src/include/mpibase.h           \
     src/include/mpibsend.h          \
-    src/include/mpich_param_vals.h  \
+    src/include/mpich_params.h  \
     src/include/mpichconfconst.h    \
     src/include/mpidbg.h            \
     src/include/mpierror.h          \
@@ -67,5 +67,5 @@ noinst_HEADERS +=                   \
     src/include/oputil.h            \
     src/include/mpiinfo.h
 
-src/include/param_vals.h: src/util/param/params.yml
+src/include/mpich_params.h: src/util/param/params.yml
 	$(top_srcdir)/maint/genparams
diff --git a/src/include/mpiimpl.h b/src/include/mpiimpl.h
index 82f5616..00efd2e 100644
--- a/src/include/mpiimpl.h
+++ b/src/include/mpiimpl.h
@@ -3716,7 +3716,7 @@ int MPID_VCR_Get_lpid(MPID_VCR vcr, int * lpid_ptr);
 #include "mpidpost.h"
 
 /* tunable parameter values */
-#include "mpich_param_vals.h"
+#include "mpich_params.h"
 
 /* Tags for point to point operations which implement collective and other
    internal operations */
diff --git a/src/mpid/common/sock/mpidu_sock.h b/src/mpid/common/sock/mpidu_sock.h
index 9b16cbf..f9a22a3 100644
--- a/src/mpid/common/sock/mpidu_sock.h
+++ b/src/mpid/common/sock/mpidu_sock.h
@@ -24,7 +24,7 @@ CPLUSPLUS_BEGIN
 #include "mpibase.h"
 #include "mpiutil.h"
 #include "mpitypedefs.h"
-#include "mpich_param_vals.h"
+#include "mpich_params.h"
 /* implementation specific header file */    
 #include "mpidu_socki.h"
 
diff --git a/src/util/param/Makefile.mk b/src/util/param/Makefile.mk
index 8c3fd7f..60c3208 100644
--- a/src/util/param/Makefile.mk
+++ b/src/util/param/Makefile.mk
@@ -6,12 +6,12 @@
 ##
 
 lib_lib at MPILIBNAME@_la_SOURCES +=   \
-    src/util/param/param_vals.c
+    src/util/param/mpich_params.c
 
 dist_noinst_DATA += src/util/param/params.yml
 
 if MAINTAINER_MODE
 # normally built by autogen.sh, but this rebuild rule is here
-$(top_srcdir)/src/util/param/param_vals.c: $(top_srcdir)/src/util/param/params.yml $(top_srcdir)/maint/genparams
+$(top_srcdir)/src/util/param/mpich_params.c: $(top_srcdir)/src/util/param/params.yml $(top_srcdir)/maint/genparams
 	( cd $(top_srcdir) && ./maint/genparams )
 endif MAINTAINER_MODE

http://git.mpich.org/mpich.git/commitdiff/255d0fb7f68e209def23ab90a3764b62f507e812

commit 255d0fb7f68e209def23ab90a3764b62f507e812
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Fri Oct 18 14:12:10 2013 -0500

    Revise genparams to gen files needed by new MPI_T
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/maint/genparams b/maint/genparams
index 36bbad0..5e7217b 100755
--- a/maint/genparams
+++ b/maint/genparams
@@ -159,111 +159,14 @@ users.
 EOT
 
 ########################################################################
-# actually process file contents, starting w/ categories
-die "missing 'categories', stopped" unless exists $params->{categories};
-
-# write enum first
-print PARAM_HDR <<EOT;
-/* parameter categories */
-enum ${ns}_category_id_t {
-EOT
-my $num_categories = scalar @{$params->{categories}};
-foreach my $cat (@{$params->{categories}}) {
-    printf PARAM_HDR "    ${uc_ns}_CATEGORY_ID_%s,\n", $cat->{name};
-}
-# then write full contents tuple
-print PARAM_HDR <<EOT;
-    ${uc_ns}_NUM_CATEGORIES
-};
-
-struct ${ns}_category_t {
-    const enum ${ns}_category_id_t id;
-    const char *name;
-    const char *description;
-};
-
-/* array of category info for runtime usage */
-extern struct ${ns}_category_t ${ns}_categories[${uc_ns}_NUM_CATEGORIES];
-
-EOT
-
-print PARAM_C <<EOT;
-/* array of category info for runtime usage */
-struct ${ns}_category_t ${ns}_categories[${uc_ns}_NUM_CATEGORIES] = {
-EOT
-foreach my $cat (@{$params->{categories}}) {
-    my $desc = $cat->{description};
-    $desc =~ s/"/\\"/g;
-    printf PARAM_C qq(    { ) .
-        qq(${uc_ns}_CATEGORY_ID_%s,\n) .
-        qq(      "%s",\n) .
-        qq(      "%s" },\n),
-        $cat->{name}, $cat->{name}, $desc;
-}
-print PARAM_C <<EOT;
-};
-
-EOT
-
-
-########################################################################
 # now the actual parameters
 die "missing 'parameters', stopped" unless exists $params->{parameters};
 
 print PARAM_HDR <<EOT;
-/* parameter values */
-enum ${ns}_id_t {
-EOT
-my $num_params = @{$params->{parameters}};
-# XXX DJG TODO collate and separate by category
-foreach my $p (@{$params->{parameters}}) {
-    printf PARAM_HDR "    ${uc_ns}_ID_%s,\n", $p->{name};
-}
-print PARAM_HDR <<EOT;
-    ${uc_ns}_NUM_PARAMS
-};
-
 /* initializes parameter values from the environment */
 int ${ns}_init_params(void);
-
 int ${ns}_finalize(void);
 
-enum ${ns}_type_t {
-    ${uc_ns}_TYPE_INVALID = 0,
-    ${uc_ns}_TYPE_INT,
-    ${uc_ns}_TYPE_DOUBLE,
-    ${uc_ns}_TYPE_BOOLEAN,
-    ${uc_ns}_TYPE_STRING,
-    ${uc_ns}_TYPE_RANGE
-};
-
-typedef struct ${ns}_param_range_val {
-    int low;
-    int high;
-} ${ns}_param_range_val_t;
-
-/* used to represent default values */
-struct ${ns}_param_default_val_t {
-    const enum ${ns}_type_t type;
-
-    /* not a union b/c of initialization portability issues */
-    const int i_val; /* also used for booleans */
-    const double d_val;
-    const char *s_val;
-    const ${ns}_param_range_val_t r_val;
-};
-
-struct ${ns}_t {
-    const enum ${ns}_id_t id;
-    const char *name;
-    const char *description;
-    const struct ${ns}_param_default_val_t default_val;
-    void *val_p; /* ptr to the actual value */
-};
-
-/* array of parameter info for runtime usage */
-extern struct ${ns}_t ${ns}_params[${uc_ns}_NUM_PARAMS];
-
 /* extern declarations for each parameter
  * (definitions in $c_file) */
 EOT
@@ -274,53 +177,8 @@ foreach my $p (@{$params->{parameters}}) {
         type2ctype($p->{type}), $p->{name};
 }
 
-print PARAM_C <<EOT;
-/* array of parameter info for runtime usage */
-struct ${ns}_t ${ns}_params[${uc_ns}_NUM_PARAMS] = {
-EOT
-
-# XXX DJG TODO collate and separate by category
-foreach my $p (@{$params->{parameters}}) {
-    my $type_enum_val = "${uc_ns}_TYPE_".uc($p->{type});
-    my ($int_val, $str_val, $double_val, $range_val) = (-1, qq(""), "0.0", "{0,0}");
-
-    if ($p->{type} eq "string") {
-        $str_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
-    }
-    elsif ($p->{type} eq "int") {
-        $int_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
-    }
-    elsif ($p->{type} eq "double") {
-        $double_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
-    }
-    elsif ($p->{type} eq "boolean") {
-        $int_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
-    }
-    elsif ($p->{type} eq "range") {
-        $range_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
-    }
-    else {
-        die "unknown type $p->{type}, stopped";
-    }
-
-    my $desc = $p->{description};
-    $desc =~ s/"/\\"/g;
-
-    printf PARAM_C qq(    { ) .
-        qq(${uc_ns}_ID_%s,\n) .
-        qq(      "%s",\n) .
-        qq(      "%s",\n) .
-        #          T    I   D   S  R     ptr
-        qq(      { %s, %s, %s, %s, %s }, %s },\n),
-        $p->{name}, $p->{name}, $desc,
-        $type_enum_val, $int_val, $double_val, $str_val, $range_val, "&${uc_ns}_".$p->{name};
-}
-
-print PARAM_C <<EOT;
-};
+print PARAM_C "/* actual storage for parameters */\n";
 
-/* actual storage for parameters */
-EOT
 foreach my $p (@{$params->{parameters}}) {
     my $default;
     if ($p->{type} eq "string") {
@@ -356,6 +214,7 @@ int ${ns}_init_params(void)
     int rc;
     const char *tmp_str;
     static int initialized = FALSE;
+    MPIR_T_cvar_value_t defaultval;
 
     /* FIXME any MT issues here? */
     if (initialized)
@@ -365,10 +224,80 @@ int ${ns}_init_params(void)
 EOT
 
 foreach my $p (@{$params->{parameters}}) {
+    my $count = 1;
+    my $mpi_dtype;
+    if ($p->{type} eq 'string') {
+        $mpi_dtype = "MPI_CHAR";
+        $count = "MPIR_PARAM_MAX_STRLEN";
+        my $str_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
+        printf PARAM_C qq(    defaultval.str = %s;\n), $str_val;
+    }
+    elsif ($p->{type} eq 'int' or $p->{type} eq 'boolean') {
+        $mpi_dtype = "MPI_INT";
+        $count = 1;
+        my $int_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
+        printf PARAM_C qq(    defaultval.d = %s;\n), $int_val;
+    }
+    elsif ($p->{type} eq 'unsigned') {
+        $mpi_dtype = "MPI_UNSINGED";
+        $count = 1;
+        my $double_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
+        printf PARAM_C qq(    defaultval.u = %s;\n), $double_val;
+    }
+    elsif ($p->{type} eq 'unsigned long') {
+        $mpi_dtype = "MPI_DOUBLE";
+        $count = 1;
+        my $double_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
+        printf PARAM_C qq(    defaultval.ul = %s;\n), $double_val;
+    }
+    elsif ($p->{type} eq 'unsigned long long') {
+        $mpi_dtype = "MPI_DOUBLE";
+        $count = 1;
+        my $double_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
+        printf PARAM_C qq(    defaultval.ull = %s;\n), $double_val;
+    }
+    elsif ($p->{type} eq 'double') {
+        $mpi_dtype = "MPI_DOUBLE";
+        $count = 1;
+        my $double_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
+        printf PARAM_C qq(    defaultval.d = %s;\n), $double_val;
+    }
+    elsif ($p->{type} eq 'range') {
+        $mpi_dtype = "MPI_INT";
+        $count = 2;
+        my $range_val = fmt_default($p->{name}, $p->{default}, $p->{defaultliteral}, $p->{type});
+        printf PARAM_C qq(    {\n);
+        printf PARAM_C qq(        MPIR_T_cvar_range_value_t tmp = %s;\n), $range_val;
+        printf PARAM_C qq(        defaultval.range = tmp;\n);
+        printf PARAM_C qq(    }\n);
+    }
+    else {
+        die "unknown type $p->{type}, stopped";
+    }
+
+    my $desc = $p->{description};
+    $desc =~ s/"/\\"/g;
+
+    printf PARAM_C qq(    MPIR_T_CVAR_REGISTER_STATIC(\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s);\n),
+        qq(        $mpi_dtype,),
+        qq(        ${uc_ns}_$p->{name}, /* name */),
+        qq(        &${uc_ns}_$p->{name}, /* address */),
+        qq(        $count, /* count */),
+        qq(        $p->{verbosity},),
+        qq(        $p->{scope},),
+        qq(        defaultval,),
+        qq(        "$p->{category}", /* category */),
+        qq(        "$desc");
+
+    if ($p->{type} eq 'string') {
+print PARAM_C <<EOT;
+    ${uc_ns}_GET_DEFAULT_STRING(${uc_ns}_$p->{name}, &tmp_str);
+EOT
+    }
+
     my $env_fn = type_to_env_fn($p->{type});
     my @env_names = ();
     my $var_name = "${uc_ns}_" . $p->{name};
-    my $var_suffix = $p->{name};
 
     # process extra envs first so the primary always wins
     push @env_names, @{$p->{'abs-alt-env'}} if $p->{'abs-alt-env'};
@@ -377,11 +306,6 @@ foreach my $p (@{$params->{parameters}}) {
     push @env_names, "${alt_ns}_" . $p->{name};
     push @env_names, "${uc_ns}_" . $p->{name};
 
-    if ($p->{type} eq 'string') {
-        print PARAM_C <<EOT;
-    ${uc_ns}_GET_DEFAULT_STRING($var_suffix, &tmp_str);
-EOT
-    }
     foreach my $env_name (@env_names) {
         # assumes rc is defined
         if ($p->{type} eq 'range') {
@@ -421,9 +345,19 @@ EOT
     print PARAM_C "\n";
 }
 
+foreach my $cat (@{$params->{categories}}) {
+    my $desc = $cat->{description};
+    $desc =~ s/"/\\"/g;
+    printf PARAM_C qq(    MPIR_T_cat_add_desc(%s\n%s);\n\n),
+        qq("$cat->{name}",),
+        qq(        "$desc");
+}
+
 print PARAM_C <<EOT;
-fn_fail:
+fn_exit:
     return mpi_errno;
+fn_fail:
+    goto fn_exit;
 }
 
 EOT
@@ -452,8 +386,10 @@ EOT
 
 
 print PARAM_C <<EOT;
-fn_fail:
+fn_exit:
     return mpi_errno;
+fn_fail:
+    goto fn_exit;
 }
 
 EOT
@@ -504,25 +440,8 @@ print PARAM_HDR <<EOT;
 
 /* helper macros for safely getting the default value of a parameter */
 EOT
-my @type_field = (
-    ['INT','i_val'],
-    ['DOUBLE','d_val'],
-    ['BOOLEAN','i_val'],
-    ['STRING','s_val'],
-    ['RANGE','r_val'],
-);
-foreach my $tuple (@type_field) {
-    my ($type,$field) = @$tuple;
-    print PARAM_HDR <<EOT;
-#define ${uc_ns}_GET_DEFAULT_${type}(p_suffix_,out_ptr_)                                               \\
-    do {                                                                                               \\
-        ${ns}_assert(${uc_ns}_TYPE_${type} == ${ns}_params[${uc_ns}_ID_##p_suffix_].default_val.type); \\
-        *(out_ptr_) = ${ns}_params[${uc_ns}_ID_##p_suffix_].default_val.${field};                      \\
-    } while (0)
-EOT
-}
-print PARAM_HDR <<EOT;
 
+print PARAM_HDR <<EOT;
 #endif /* $hdr_guard */
 EOT
 close(PARAM_HDR);
@@ -549,7 +468,7 @@ sub type2ctype {
         'double'  => 'double',
         'string'  => 'char *',
         'boolean' => 'int',
-        'range'   => "${ns}_param_range_val_t",
+        'range'   => "MPIR_T_cvar_range_value_t",
     );
     die "unknown type '$type', stopped" unless exists $typemap{$type};
     return $typemap{$type};

http://git.mpich.org/mpich.git/commitdiff/127ee1d283c738847e35120d462686b89845521e

commit 127ee1d283c738847e35120d462686b89845521e
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Thu Oct 24 16:10:28 2013 -0500

    Add missing info of parameters (cvars)
    
    This makes parameters (cvars) conform to MPI_T standard.
    However, probably the info is too conservative.
    Need further tuning the info.
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/src/util/param/params.yml b/src/util/param/params.yml
index b442639..9ea8d81 100644
--- a/src/util/param/params.yml
+++ b/src/util/param/params.yml
@@ -60,6 +60,8 @@ parameters:
       type        : int
       default     : 256
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         the short message algorithm will be used if the per-destination
         message size (sendcount*size(sendtype)) is <= this value
@@ -69,6 +71,8 @@ parameters:
       type        : int
       default     : 32768
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         the medium message algorithm will be used if the per-destination
         message size (sendcount*size(sendtype)) is <= this value and
@@ -79,6 +83,8 @@ parameters:
       type        : int
       default     : 32
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         max no. of irecvs/isends posted at a time in some alltoall
         algorithms. Setting it to 0 causes all irecvs/isends to be
@@ -89,6 +95,8 @@ parameters:
       type        : int
       default     : 524288
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         the long message algorithm will be used if the operation is commutative
         and the send buffer size is >= this value (in bytes)
@@ -98,6 +106,8 @@ parameters:
       type        : int
       default     : 8
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         the minimum number of processes in a communicator to use a non-binomial
         broadcast algorithm
@@ -107,6 +117,8 @@ parameters:
       type        : int
       default     : 12288
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         the short message algorithm will be used if the send buffer size is <
         this value (in bytes)
@@ -116,6 +128,8 @@ parameters:
       type        : int
       default     : 524288
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         the long message algorithm will be used if the send buffer size is >=
         this value (in bytes)
@@ -125,6 +139,8 @@ parameters:
       type        : int
       default     : 81920
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         For MPI_Allgather and MPI_Allgatherv, the short message algorithm will
         be used if the send buffer size is < this value (in bytes).
@@ -134,6 +150,8 @@ parameters:
       type        : int
       default     : 524288
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         For MPI_Allgather and MPI_Allgatherv, the long message algorithm will be
         used if the send buffer size is >= this value (in bytes)
@@ -143,6 +161,8 @@ parameters:
       type        : int
       default     : 2048
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         the short message algorithm will be used if the send buffer size is <=
         this value (in bytes)
@@ -152,6 +172,8 @@ parameters:
       type        : int
       default     : 2048
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         the short message algorithm will be used if the send buffer size is <=
         this value (in bytes)
@@ -161,6 +183,8 @@ parameters:
       type        : int
       default     : 1024
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         use a temporary buffer for intracommunicator MPI_Gather if the send
         buffer size is < this value (in bytes)
@@ -170,6 +194,8 @@ parameters:
       type        : int
       default     : 2048
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         use the short message algorithm for intercommunicator MPI_Gather if the
         send buffer size is < this value (in bytes)
@@ -179,6 +205,8 @@ parameters:
       type        : int
       default     : 32
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Use Ssend (synchronous send) for intercommunicator MPI_Gatherv if the
         "group B" size is >= this value.  Specifying "-1" always avoids using
@@ -190,6 +218,8 @@ parameters:
       type        : int
       default     : 2048
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         use the short message algorithm for intercommunicator MPI_Scatter if the
         send buffer size is < this value (in bytes)
@@ -199,6 +229,8 @@ parameters:
       type        : int
       default     : 32768
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         The smallest message size that will be used for the pipelined, large-message,
         ring algorithm in the MPI_Allgatherv implementation.
@@ -208,6 +240,8 @@ parameters:
       type        : boolean
       default     : true
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Enable SMP aware collective communication.
 
@@ -216,6 +250,8 @@ parameters:
       type        : boolean
       default     : true
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Enable SMP aware barrier.
 
@@ -224,6 +260,8 @@ parameters:
       type        : boolean
       default     : true
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Enable SMP aware broadcast.
 
@@ -232,6 +270,8 @@ parameters:
       type        : int
       default     : 0
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Maximum message size for which SMP-aware broadcast is used.  A
         value of '0' uses SMP-aware broadcast for all message sizes.
@@ -241,6 +281,8 @@ parameters:
       type        : boolean
       default     : true
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Enable SMP aware reduce.
 
@@ -249,6 +291,8 @@ parameters:
       type        : int
       default     : 0
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Maximum message size for which SMP-aware reduce is used.  A
         value of '0' uses SMP-aware reduce for all message sizes.
@@ -258,6 +302,8 @@ parameters:
       type        : boolean
       default     : true
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Enable SMP aware allreduce.
 
@@ -266,6 +312,8 @@ parameters:
       type        : int
       default     : 0
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Maximum message size for which SMP-aware allreduce is used.  A
         value of '0' uses SMP-aware allreduce for all message sizes.
@@ -277,6 +325,8 @@ parameters:
       type        : boolean
       default     : true
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Use qsort(3) in the implementation of MPI_Comm_split instead of bubble sort.
 
@@ -287,6 +337,8 @@ parameters:
       type        : boolean
       default     : false
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         If true, causes processes to wait in MPI_Init and
         MPI_Initthread for a debugger to be attached.  Once the
@@ -299,6 +351,8 @@ parameters:
       type        : int
       default     : 64
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Size of the "MPIR" debugger interface proctable (process table).
 
@@ -307,6 +361,8 @@ parameters:
       type        : boolean
       default     : true
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         If true, dump the proctable entries at MPIR_WaitForDebugger-time.
         (currently compile-time disabled by "#if 0")
@@ -318,6 +374,8 @@ parameters:
       type        : boolean
       default     : true
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         DEPRECATED! Will be removed in MPICH-3.2
         Collectives called on a communicator with a failed process
@@ -333,6 +391,8 @@ parameters:
       type        : boolean
       default     : false
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         If true, MPI will call MPI_Abort at MPI_Finalize if any MPI object
         handles have been leaked.  For example, if MPI_Comm_dup is called
@@ -349,6 +409,8 @@ parameters:
       type        : int
       default     : 2
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         The MPIR_PARAM_CTXID_EAGER_SIZE environment variable allows you to
         specify how many words in the context ID mask will be set aside
@@ -360,6 +422,8 @@ parameters:
       type        : boolean
       default     : false
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         If set to true, MPICH will initiate an additional thread to
         make asynchronous progress on all communication operations
@@ -379,6 +443,8 @@ parameters:
       type        : string
       default     : "MPI_THREAD_SINGLE"
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Sets the default thread level to use when using MPI_INIT.
 
@@ -389,6 +455,8 @@ parameters:
       type        : boolean
       default     : true
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         If true, list any memory that was allocated by MPICH and that
         remains allocated when MPI_Finalize completes.
@@ -398,6 +466,8 @@ parameters:
       type        : boolean
       default     : true
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         If true, perform checks for errors, typically to verify valid inputs
         to MPI routines.  Only effective when MPICH is configured with
@@ -408,6 +478,8 @@ parameters:
       type        : boolean
       default     : true
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         If true, print an error stack trace at error handling time.
 
@@ -416,6 +488,8 @@ parameters:
       type        : int
       default     : 0
       class       : device
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         If >0, truncate error stack output lines this many characters
         wide.  If 0, do not truncate, and if <0 use a sensible default.
@@ -427,6 +501,8 @@ parameters:
       type        : boolean
       default     : true
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Use the immediate accumulate optimization
 
@@ -435,6 +511,8 @@ parameters:
       type        : int
       default     : 4000
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Threshold at which the RMA implementation attempts to complete requests
         while completing RMA operations and while using the lazy synchonization
@@ -446,6 +524,8 @@ parameters:
       type        : int
       default     : 128
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Threshold for the number of new requests since the last attempt to
         complete pending requests.  Higher values can increase performance,
@@ -457,6 +537,8 @@ parameters:
       type        : boolean
       default     : false
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Issue a request for the passive target RMA lock immediately.  Default
         behavior is to defer the lock request until the call to MPI_Win_unlock.
@@ -466,6 +548,8 @@ parameters:
       type        : boolean
       default     : true
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Enable/disable an optimization that merges lock, op, and unlock
         messages, for single-operation passive target epochs.
@@ -477,6 +561,8 @@ parameters:
       type        : boolean
       default     : false
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         If true, force all processes to operate as though all processes
         are located on another node.  For example, this disables shared
@@ -489,6 +575,8 @@ parameters:
       type        : boolean
       default     : false
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         If true, odd procs on a node are seen as local to each other, and even
         procs on a node are seen as local to each other.  Used for debugging on
@@ -501,6 +589,8 @@ parameters:
       type        : string
       defaultliteral : NULL
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         If non-NULL, this parameter specifies the IP address that
         other processes should use when connecting to this process.
@@ -513,6 +603,8 @@ parameters:
       type        : int
       default     : 131072
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         This parameter controls the message size at which CH3 switches
         from eager to rendezvous mode.
@@ -525,6 +617,8 @@ parameters:
       type        : range
       default     : "0:0"
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         The MPIR_PARAM_CH3_PORT_RANGE environment variable allows you to
         specify the range of TCP ports to be used by the process
@@ -538,6 +632,8 @@ parameters:
       type        : int
       default     : 1000
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         When MPICH is in a busy waiting loop, it will periodically
         call a function to yield the processor.  This parameter sets
@@ -549,6 +645,8 @@ parameters:
       type        : int
       default     : 2097152
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Messages larger than this size will use the "dma" (knem)
         intranode LMT implementation, if it is enabled and available.
@@ -558,6 +656,8 @@ parameters:
       type        : string
       default     : ""
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         If non-empty, this parameter specifies which network module
         should be used for communication.
@@ -567,6 +667,8 @@ parameters:
       type        : int
       default     : -1
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         This parameter controls the message size at which Nemesis
         switches from eager to rendezvous mode for shared memory.
@@ -578,6 +680,8 @@ parameters:
       type        : int
       default     : -2
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         This parameter controls the message size at which Nemesis
         switches from eager to rendezvous mode for ready-send
@@ -590,6 +694,8 @@ parameters:
       type        : boolean
       default     : false
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         If true, enables checkpointing support and returns an error if
         checkpointing library cannot be initialized.
@@ -603,6 +709,8 @@ parameters:
       type        : string
       defaultliteral : NULL
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         If non-NULL, this parameter specifies which pseudo-ethernet
         interface the tcp netmod should use (e.g., "eth1", "ib0").
@@ -616,6 +724,8 @@ parameters:
       type        : int
       default     : 10
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         This parameter controls the number of times to retry the
         gethostbyname() function before giving up.
@@ -627,6 +737,8 @@ parameters:
       type        : boolean
       default     : true
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         If set to false, communication override functionality will be
         disabled for netmods that provide the override feature.
@@ -642,6 +754,8 @@ parameters:
       type        : string
       defaultliteral : NULL
       class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
       description : >-
         Sets the directory to use for MPI service publishing in the
         file nameserv implementation.  Allows the user to override

http://git.mpich.org/mpich.git/commitdiff/4c5f24c85b7bb51d53b9ddc2b8332cecfb948fe2

commit 4c5f24c85b7bb51d53b9ddc2b8332cecfb948fe2
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Thu Oct 24 16:08:54 2013 -0500

    Capitalize parameter category names
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/src/util/param/params.yml b/src/util/param/params.yml
index c300e96..b442639 100644
--- a/src/util/param/params.yml
+++ b/src/util/param/params.yml
@@ -19,43 +19,43 @@
 # will allow the device to override the default value.
 
 categories:
-    - name        : collective
+    - name        : COLLECTIVE
       description : parameters that control collective communication behavior
-    - name        : communicator
+    - name        : COMMUNICATOR
       description : parameters that control communicator construction and operation
-    - name        : pt2pt
+    - name        : PT2PT
       description : parameters that control point-to-point communication behavior
-    - name        : rma
+    - name        : RMA
       description : parameters that control RMA communicatin behavior
-    - name        : intranode
+    - name        : INTRANODE
       description : intranode communication parameters
-    - name        : developer
+    - name        : DEVELOPER
       description : useful for developers working on MPICH itself
-    - name        : memory
+    - name        : MEMORY
       description : affects memory allocation and usage, including MPI object handles
-    - name        : error_handling
+    - name        : ERROR_HANDLING
       description : parameters that control error handling behavior (stack traces, aborts, etc)
-    - name        : debugger
+    - name        : DEBUGGER
       description : parameters relevant to the "MPIR" debugger interface
-    - name        : checkpointing
+    - name        : CHECKPOINTING
       description : parameters relevant to checkpointing
-    - name        : fault_tolerance
+    - name        : FAULT_TOLERANCE
       description : parameters that control fault tolerance behavior
-    - name        : threads
+    - name        : THREADS
       description : multi-threading parameters
-    - name        : ch3
+    - name        : CH3
       description : parameters that control behavior of ch3
-    - name        : nemesis
+    - name        : NEMESIS
       description : "parameters that control behavior of the ch3:nemesis channel"
-    - name        : sockets
+    - name        : SOCKETS
       description : control socket parameters
-    - name        : process_manager
+    - name        : PROCESS_MANAGER
       description : parameters that control the client-side process manager code
 
 parameters:
     ##############################################################
     # collective parameters
-    - category    : collective
+    - category    : COLLECTIVE
       name        : ALLTOALL_SHORT_MSG_SIZE
       type        : int
       default     : 256
@@ -64,7 +64,7 @@ parameters:
         the short message algorithm will be used if the per-destination
         message size (sendcount*size(sendtype)) is <= this value
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : ALLTOALL_MEDIUM_MSG_SIZE
       type        : int
       default     : 32768
@@ -74,7 +74,7 @@ parameters:
         message size (sendcount*size(sendtype)) is <= this value and
         larger than ALLTOALL_SHORT_MSG_SIZE
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : ALLTOALL_THROTTLE
       type        : int
       default     : 32
@@ -84,7 +84,7 @@ parameters:
         algorithms. Setting it to 0 causes all irecvs/isends to be
         posted at once.
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : REDSCAT_COMMUTATIVE_LONG_MSG_SIZE
       type        : int
       default     : 524288
@@ -93,7 +93,7 @@ parameters:
         the long message algorithm will be used if the operation is commutative
         and the send buffer size is >= this value (in bytes)
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : BCAST_MIN_PROCS
       type        : int
       default     : 8
@@ -102,7 +102,7 @@ parameters:
         the minimum number of processes in a communicator to use a non-binomial
         broadcast algorithm
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : BCAST_SHORT_MSG_SIZE
       type        : int
       default     : 12288
@@ -111,7 +111,7 @@ parameters:
         the short message algorithm will be used if the send buffer size is <
         this value (in bytes)
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : BCAST_LONG_MSG_SIZE
       type        : int
       default     : 524288
@@ -120,7 +120,7 @@ parameters:
         the long message algorithm will be used if the send buffer size is >=
         this value (in bytes)
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : ALLGATHER_SHORT_MSG_SIZE
       type        : int
       default     : 81920
@@ -129,7 +129,7 @@ parameters:
         For MPI_Allgather and MPI_Allgatherv, the short message algorithm will
         be used if the send buffer size is < this value (in bytes).
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : ALLGATHER_LONG_MSG_SIZE
       type        : int
       default     : 524288
@@ -138,7 +138,7 @@ parameters:
         For MPI_Allgather and MPI_Allgatherv, the long message algorithm will be
         used if the send buffer size is >= this value (in bytes)
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : REDUCE_SHORT_MSG_SIZE
       type        : int
       default     : 2048
@@ -147,7 +147,7 @@ parameters:
         the short message algorithm will be used if the send buffer size is <=
         this value (in bytes)
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : ALLREDUCE_SHORT_MSG_SIZE
       type        : int
       default     : 2048
@@ -156,7 +156,7 @@ parameters:
         the short message algorithm will be used if the send buffer size is <=
         this value (in bytes)
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : GATHER_VSMALL_MSG_SIZE
       type        : int
       default     : 1024
@@ -165,7 +165,7 @@ parameters:
         use a temporary buffer for intracommunicator MPI_Gather if the send
         buffer size is < this value (in bytes)
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : GATHER_INTER_SHORT_MSG_SIZE
       type        : int
       default     : 2048
@@ -174,7 +174,7 @@ parameters:
         use the short message algorithm for intercommunicator MPI_Gather if the
         send buffer size is < this value (in bytes)
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : GATHERV_INTER_SSEND_MIN_PROCS
       type        : int
       default     : 32
@@ -185,7 +185,7 @@ parameters:
         Ssend.  For backwards compatibility, specifying "0" uses the default
         value.
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : SCATTER_INTER_SHORT_MSG_SIZE
       type        : int
       default     : 2048
@@ -194,7 +194,7 @@ parameters:
         use the short message algorithm for intercommunicator MPI_Scatter if the
         send buffer size is < this value (in bytes)
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : ALLGATHERV_PIPELINE_MSG_SIZE
       type        : int
       default     : 32768
@@ -203,7 +203,7 @@ parameters:
         The smallest message size that will be used for the pipelined, large-message,
         ring algorithm in the MPI_Allgatherv implementation.
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : ENABLE_SMP_COLLECTIVES
       type        : boolean
       default     : true
@@ -211,7 +211,7 @@ parameters:
       description : >-
         Enable SMP aware collective communication.
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : ENABLE_SMP_BARRIER
       type        : boolean
       default     : true
@@ -219,7 +219,7 @@ parameters:
       description : >-
         Enable SMP aware barrier.
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : ENABLE_SMP_BCAST
       type        : boolean
       default     : true
@@ -227,7 +227,7 @@ parameters:
       description : >-
         Enable SMP aware broadcast.
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : MAX_SMP_BCAST_MSG_SIZE
       type        : int
       default     : 0
@@ -236,7 +236,7 @@ parameters:
         Maximum message size for which SMP-aware broadcast is used.  A
         value of '0' uses SMP-aware broadcast for all message sizes.
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : ENABLE_SMP_REDUCE
       type        : boolean
       default     : true
@@ -244,7 +244,7 @@ parameters:
       description : >-
         Enable SMP aware reduce.
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : MAX_SMP_REDUCE_MSG_SIZE
       type        : int
       default     : 0
@@ -253,7 +253,7 @@ parameters:
         Maximum message size for which SMP-aware reduce is used.  A
         value of '0' uses SMP-aware reduce for all message sizes.
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : ENABLE_SMP_ALLREDUCE
       type        : boolean
       default     : true
@@ -261,7 +261,7 @@ parameters:
       description : >-
         Enable SMP aware allreduce.
 
-    - category    : collective
+    - category    : COLLECTIVE
       name        : MAX_SMP_ALLREDUCE_MSG_SIZE
       type        : int
       default     : 0
@@ -272,7 +272,7 @@ parameters:
 
     ##############################################################
     # communicator parameters
-    - category    : communicator
+    - category    : COMMUNICATOR
       name        : COMM_SPLIT_USE_QSORT
       type        : boolean
       default     : true
@@ -282,7 +282,7 @@ parameters:
 
   ##############################################################
     # debugging parameters
-    - category    : debugger
+    - category    : DEBUGGER
       name        : DEBUG_HOLD
       type        : boolean
       default     : false
@@ -294,7 +294,7 @@ parameters:
         in order to allow the process to continue (e.g., in gdb, "set
         hold=0").
 
-    - category    : debugger
+    - category    : DEBUGGER
       name        : PROCTABLE_SIZE
       type        : int
       default     : 64
@@ -302,7 +302,7 @@ parameters:
       description : >-
         Size of the "MPIR" debugger interface proctable (process table).
 
-    - category    : debugger
+    - category    : DEBUGGER
       name        : PROCTABLE_PRINT
       type        : boolean
       default     : true
@@ -313,7 +313,7 @@ parameters:
 
   ##############################################################
     # fault-tolerance parameters
-    - category    : fault_tolerance
+    - category    : FAULT_TOLERANCE
       name        : ENABLE_COLL_FT_RET
       type        : boolean
       default     : true
@@ -328,7 +328,7 @@ parameters:
 
   ##############################################################
     # memory parameters
-    - category    : memory
+    - category    : MEMORY
       name        : ABORT_ON_LEAKED_HANDLES
       type        : boolean
       default     : false
@@ -344,7 +344,7 @@ parameters:
 
   ############################
     # threads parameters
-    - category    : threads
+    - category    : THREADS
       name        : CTXID_EAGER_SIZE
       type        : int
       default     : 2
@@ -355,7 +355,7 @@ parameters:
         for the eager allocation protocol.  If the application is running
         out of context IDs, reducing this value may help.
 
-    - category    : threads
+    - category    : THREADS
       name        : ASYNC_PROGRESS
       type        : boolean
       default     : false
@@ -374,7 +374,7 @@ parameters:
         highly system dependent but may be substantial in some cases,
         hence this recommendation.
 
-    - category    : threads
+    - category    : THREADS
       name        : DEFAULT_THREAD_LEVEL
       type        : string
       default     : "MPI_THREAD_SINGLE"
@@ -384,7 +384,7 @@ parameters:
 
     ##############################################################
     # other MPI-level parameters
-    - category    : developer
+    - category    : DEVELOPER
       name        : MEMDUMP
       type        : boolean
       default     : true
@@ -393,7 +393,7 @@ parameters:
         If true, list any memory that was allocated by MPICH and that
         remains allocated when MPI_Finalize completes.
 
-    - category    : error_handling
+    - category    : ERROR_HANDLING
       name        : ERROR_CHECKING
       type        : boolean
       default     : true
@@ -403,7 +403,7 @@ parameters:
         to MPI routines.  Only effective when MPICH is configured with
         --enable-error-checking=runtime .
 
-    - category    : error_handling
+    - category    : ERROR_HANDLING
       name        : PRINT_ERROR_STACK
       type        : boolean
       default     : true
@@ -411,7 +411,7 @@ parameters:
       description : >-
         If true, print an error stack trace at error handling time.
 
-    - category    : error_handling
+    - category    : ERROR_HANDLING
       name        : CHOP_ERROR_STACK
       type        : int
       default     : 0
@@ -422,7 +422,7 @@ parameters:
 
     ##############################################################
     # ch3 parameters
-    - category    : ch3
+    - category    : CH3
       name        : CH3_RMA_ACC_IMMED
       type        : boolean
       default     : true
@@ -430,7 +430,7 @@ parameters:
       description : >-
         Use the immediate accumulate optimization
 
-    - category    : ch3
+    - category    : CH3
       name        : CH3_RMA_NREQUEST_THRESHOLD
       type        : int
       default     : 4000
@@ -441,7 +441,7 @@ parameters:
         approach.  Change this value if programs fail because they run out of
         requests or other internal resources
 
-    - category    : ch3
+    - category    : CH3
       name        : CH3_RMA_NREQUEST_NEW_THRESHOLD
       type        : int
       default     : 128
@@ -452,7 +452,7 @@ parameters:
         but may run the risk of exceeding the available number of requests
         or other internal resources.
 
-    - category    : ch3
+    - category    : CH3
       name        : CH3_RMA_LOCK_IMMED
       type        : boolean
       default     : false
@@ -461,7 +461,7 @@ parameters:
         Issue a request for the passive target RMA lock immediately.  Default
         behavior is to defer the lock request until the call to MPI_Win_unlock.
 
-    - category    : ch3
+    - category    : CH3
       name        : CH3_RMA_MERGE_LOCK_OP_UNLOCK
       type        : boolean
       default     : true
@@ -470,7 +470,7 @@ parameters:
         Enable/disable an optimization that merges lock, op, and unlock
         messages, for single-operation passive target epochs.
 
-    - category    : ch3
+    - category    : CH3
       name        : CH3_NOLOCAL
       alt-env     :
                     - CH3_NO_LOCAL
@@ -482,7 +482,7 @@ parameters:
         are located on another node.  For example, this disables shared
         memory communication hierarchical collectives.
 
-    - category    : ch3
+    - category    : CH3
       name        : CH3_ODD_EVEN_CLIQUES
       alt-env     :
                     - CH3_EVEN_ODD_CLIQUES
@@ -494,7 +494,7 @@ parameters:
         procs on a node are seen as local to each other.  Used for debugging on
         a single machine.
 
-    - category    : ch3
+    - category    : CH3
       name        : CH3_INTERFACE_HOSTNAME
       alt-env     :
                     - INTERFACE_HOSTNAME
@@ -508,7 +508,7 @@ parameters:
         MPIR_PARAM_CH3_NETWORK_IFACE parameter and it is an error to set them
         both.
 
-    - category    : ch3
+    - category    : CH3
       name        : CH3_EAGER_MAX_MSG_SIZE
       type        : int
       default     : 131072
@@ -517,7 +517,7 @@ parameters:
         This parameter controls the message size at which CH3 switches
         from eager to rendezvous mode.
 
-    - category    : ch3
+    - category    : CH3
       name        : CH3_PORT_RANGE
       alt-env     :
                     - PORTRANGE
@@ -533,7 +533,7 @@ parameters:
 
     ##############################################################
     # nemesis parameters
-    - category    : nemesis
+    - category    : NEMESIS
       name        : NEMESIS_POLLS_BEFORE_YIELD
       type        : int
       default     : 1000
@@ -544,7 +544,7 @@ parameters:
         the number of loops before the yield function is called.  A
         value of 0 disables yielding.
 
-    - category    : nemesis
+    - category    : NEMESIS
       name        : NEMESIS_LMT_DMA_THRESHOLD
       type        : int
       default     : 2097152
@@ -553,7 +553,7 @@ parameters:
         Messages larger than this size will use the "dma" (knem)
         intranode LMT implementation, if it is enabled and available.
 
-    - category    : nemesis
+    - category    : NEMESIS
       name        : NEMESIS_NETMOD
       type        : string
       default     : ""
@@ -562,7 +562,7 @@ parameters:
         If non-empty, this parameter specifies which network module
         should be used for communication.
 
-    - category    : nemesis
+    - category    : NEMESIS
       name        : NEMESIS_SHM_EAGER_MAX_SZ
       type        : int
       default     : -1
@@ -573,7 +573,7 @@ parameters:
         If this parameter is set to -1, then Nemesis will choose
         an appropriate value.
 
-    - category    : nemesis
+    - category    : NEMESIS
       name        : NEMESIS_SHM_READY_EAGER_MAX_SZ
       type        : int
       default     : -2
@@ -585,7 +585,7 @@ parameters:
         will always be sent eagerly.  If this parameter is set to -2,
         then Nemesis will choose an appropriate value.
 
-    - category    : nemesis
+    - category    : NEMESIS
       name        : NEMESIS_ENABLE_CKPOINT
       type        : boolean
       default     : false
@@ -596,7 +596,7 @@ parameters:
 
     ##############################################################
     # nemesis TCP parameters
-    - category    : nemesis
+    - category    : NEMESIS
       name        : NEMESIS_TCP_NETWORK_IFACE
       alt-env     :
                     - NETWORK_IFACE
@@ -611,7 +611,7 @@ parameters:
         MPIR_PARAM_CH3_INTERFACE_HOSTNAME parameter and it is an error to set
         them both.
 
-    - category    : nemesis
+    - category    : NEMESIS
       name        : NEMESIS_TCP_HOST_LOOKUP_RETRIES
       type        : int
       default     : 10
@@ -622,7 +622,7 @@ parameters:
 
     ##############################################################
     # nemesis portals parameters
-    - category    : nemesis
+    - category    : NEMESIS
       name        : NEMESIS_PORTALS_COMM_OVERRIDES
       type        : boolean
       default     : true
@@ -635,7 +635,7 @@ parameters:
 
     ##############################################################
     # process manager parameters
-    - category    : process_manager
+    - category    : PROCESS_MANAGER
       name        : NAMESERV_FILE_PUBDIR
       alt-env     :
                     - NAMEPUB_DIR

http://git.mpich.org/mpich.git/commitdiff/e39e4d2e87b47f1497e39fd626315fa3c4f2012f

commit e39e4d2e87b47f1497e39fd626315fa3c4f2012f
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Fri Oct 18 15:20:25 2013 -0500

    Adapt old ch3 pvars to new MPI_T impl
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/src/mpid/ch3/src/ch3u_eager.c b/src/mpid/ch3/src/ch3u_eager.c
index 2ca4e78..80f1acb 100644
--- a/src/mpid/ch3/src/ch3u_eager.c
+++ b/src/mpid/ch3/src/ch3u_eager.c
@@ -436,7 +436,7 @@ int MPIDI_CH3_PktHandler_EagerShortSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
                a buffer that we've allocated). */
 	    /* printf( "Allocating into tmp\n" ); fflush(stdout); */
 	    recv_data_sz = rreq->dev.recv_data_sz;
-            MPIR_T_ADD(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, recv_data_sz);
+        MPIR_T_PVAR_LEVEL_INC(RECVQ, unexpected_recvq_buffer_size, recv_data_sz);
 	    rreq->dev.tmpbuf = MPIU_Malloc(recv_data_sz);
 	    if (!rreq->dev.tmpbuf) {
 		MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
diff --git a/src/mpid/ch3/src/ch3u_handle_connection.c b/src/mpid/ch3/src/ch3u_handle_connection.c
index 6db836a..fbbe3cb 100644
--- a/src/mpid/ch3/src/ch3u_handle_connection.c
+++ b/src/mpid/ch3/src/ch3u_handle_connection.c
@@ -10,6 +10,7 @@
 #else
 #include "pmi.h"
 #endif
+#undef utarray_oom
 #define utarray_oom() do { goto fn_oom; } while (0)
 #include "mpiu_utarray.h"
 
diff --git a/src/mpid/ch3/src/ch3u_handle_recv_pkt.c b/src/mpid/ch3/src/ch3u_handle_recv_pkt.c
index 8934f9e..30a84fe 100644
--- a/src/mpid/ch3/src/ch3u_handle_recv_pkt.c
+++ b/src/mpid/ch3/src/ch3u_handle_recv_pkt.c
@@ -274,7 +274,7 @@ int MPIDI_CH3U_Receive_data_unexpected(MPID_Request * rreq, char *buf, MPIDI_msg
     }
 
     if (MPIDI_Request_get_msg_type(rreq) == MPIDI_REQUEST_EAGER_MSG)
-        MPIR_T_ADD(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
+        MPIR_T_PVAR_LEVEL_INC(RECVQ, unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
 
     rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_UnpackUEBufComplete;
 
diff --git a/src/mpid/ch3/src/ch3u_recvq.c b/src/mpid/ch3/src/ch3u_recvq.c
index 5bef548..c2ab01f 100644
--- a/src/mpid/ch3/src/ch3u_recvq.c
+++ b/src/mpid/ch3/src/ch3u_recvq.c
@@ -59,18 +59,6 @@ MPID_Request ** const MPID_Recvq_posted_head_ptr     = &recvq_posted_head;
 MPID_Request ** const MPID_Recvq_unexpected_head_ptr = &recvq_unexpected_head;
 #endif
 
-/* Always define these, since our macros will always generate references to
- * them.  Every compiler we have tested optimizes these "if(0){...}" code paths
- * away, even at "-O0". */
-static unsigned int posted_qlen = 0;
-static unsigned int unexpected_qlen = 0;
-static MPI_Aint posted_recvq_match_attempts = 0;
-static MPI_Aint unexpected_recvq_match_attempts = 0;
-static double time_failed_matching_postedq = 0.0;
-static double time_matching_unexpectedq = 0.0;
-uint64_t MPIDI_CH3I_unexpected_recvq_buffer_size = 0;    /* used in ch3u_eager.c and ch3u_handle_recv_pkt.c */
-
-
 /* If the MPIDI_Message_match structure fits into a pointer size, we
  * can directly work on it */
 /* MATCH_WITH_NO_MASK compares the match values without masking
@@ -104,10 +92,15 @@ uint64_t MPIDI_CH3I_unexpected_recvq_buffer_size = 0;    /* used in ch3u_eager.c
       ((match1).parts.context_id == (match2).parts.context_id)))
 
 
-/* will be invoked to populate the custom parts of pvar_handle objects */
-MPIR_T_SIMPLE_HANDLE_CREATOR(simple_aint_creator, MPI_Aint, 1)
-MPIR_T_SIMPLE_HANDLE_CREATOR(simple_uint_creator, unsigned int, 1)
-MPIR_T_SIMPLE_HANDLE_CREATOR(simple_double_creator, double, 1)
+MPIR_T_PVAR_UINT_LEVEL_DECL_STATIC(RECVQ, posted_recvq_length);
+MPIR_T_PVAR_UINT_LEVEL_DECL_STATIC(RECVQ, unexpected_recvq_length);
+MPIR_T_PVAR_ULONG2_COUNTER_DECL_STATIC(RECVQ, posted_recvq_match_attempts);
+MPIR_T_PVAR_ULONG2_COUNTER_DECL_STATIC(RECVQ, unexpected_recvq_match_attempts);
+MPIR_T_PVAR_DOUBLE_TIMER_DECL_STATIC(RECVQ, time_failed_matching_postedq);
+MPIR_T_PVAR_DOUBLE_TIMER_DECL_STATIC(RECVQ, time_matching_unexpectedq);
+
+/* used in ch3u_eager.c and ch3u_handle_recv_pkt.c */
+MPIR_T_PVAR_ULONG2_LEVEL_DECL(RECVQ, unexpected_recvq_buffer_size);
 
 #undef FUNCNAME
 #define FUNCNAME MPIDI_CH3U_Recvq_init
@@ -116,126 +109,79 @@ MPIR_T_SIMPLE_HANDLE_CREATOR(simple_double_creator, double, 1)
 int MPIDI_CH3U_Recvq_init(void)
 {
     int mpi_errno = MPI_SUCCESS;
-#if ENABLE_RECVQ_STATISTICS
-    int idx = -1;
-    mpi_errno = MPIR_T_pvar_add("posted_recvq_length",
-                                MPI_T_VERBOSITY_USER_DETAIL,
-                                MPI_T_PVAR_CLASS_LEVEL,
-                                MPI_UNSIGNED,
-                                MPI_T_ENUM_NULL,
-                                "length of the posted message receive queue",
-                                MPI_T_BIND_NO_OBJECT,
-                                /*readonly=*/TRUE,
-                                /*continuous=*/TRUE,
-                                /*atomic=*/FALSE,
-                                MPIR_T_PVAR_IMPL_SIMPLE,
-                                /*var_state=*/&posted_qlen,
-                                &simple_uint_creator,
-                                &idx);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
-    mpi_errno = MPIR_T_pvar_add("unexpected_recvq_length",
-                                MPI_T_VERBOSITY_USER_DETAIL,
-                                MPI_T_PVAR_CLASS_LEVEL,
-                                MPI_UNSIGNED,
-                                MPI_T_ENUM_NULL,
-                                "length of the unexpected message receive queue",
-                                MPI_T_BIND_NO_OBJECT,
-                                /*readonly=*/TRUE,
-                                /*continuous=*/TRUE,
-                                /*atomic=*/FALSE,
-                                MPIR_T_PVAR_IMPL_SIMPLE,
-                                /*var_state=*/&unexpected_qlen,
-                                &simple_uint_creator,
-                                &idx);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
-    /* posted receive queue failed matches */
-    mpi_errno = MPIR_T_pvar_add("posted_recvq_match_attempts",
-                                MPI_T_VERBOSITY_USER_DETAIL,
-                                MPI_T_PVAR_CLASS_COUNTER,
-                                MPI_AINT,
-                                MPI_T_ENUM_NULL,
-                                "number of search passes on the message receive queue",
-                                MPI_T_BIND_NO_OBJECT,
-                                /*readonly=*/FALSE,
-                                /*continuous=*/TRUE,
-                                /*atomic=*/FALSE,
-                                MPIR_T_PVAR_IMPL_SIMPLE,
-                                /*var_state=*/&posted_recvq_match_attempts,
-                                &simple_aint_creator,
-                                &idx);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
-    /* unexpected receive queue failed matches */
-    mpi_errno = MPIR_T_pvar_add("unexpected_recvq_match_attempts",
-                                MPI_T_VERBOSITY_USER_DETAIL,
-                                MPI_T_PVAR_CLASS_COUNTER,
-                                MPI_AINT,
-                                MPI_T_ENUM_NULL,
-                                "number of search passes on the message receive queue",
-                                MPI_T_BIND_NO_OBJECT,
-                                /*readonly=*/FALSE,
-                                /*continuous=*/TRUE,
-                                /*atomic=*/FALSE,
-                                MPIR_T_PVAR_IMPL_SIMPLE,
-                                /*var_state=*/&unexpected_recvq_match_attempts,
-                                &simple_aint_creator,
-                                &idx);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
-    /* time spent unsuccessfully trying to match incoming message with posted receives */
-    mpi_errno = MPIR_T_pvar_add("time_failed_matching_postedq",
-                                MPI_T_VERBOSITY_USER_DETAIL,
-                                MPI_T_PVAR_CLASS_TIMER,
-                                MPI_DOUBLE,
-                                MPI_T_ENUM_NULL,
-                                "total time spent on unsuccessful search passes on the posted receives queue",
-                                MPI_T_BIND_NO_OBJECT,
-                                /*readonly=*/FALSE,
-                                /*continuous=*/TRUE,
-                                /*atomic=*/FALSE,
-                                MPIR_T_PVAR_IMPL_SIMPLE,
-                                /*var_state=*/&time_failed_matching_postedq,
-                                &simple_double_creator,
-                                &idx);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
-    /* time spent trying to match a posted receive with messages in the unexpected queue */
-    mpi_errno = MPIR_T_pvar_add("time_matching_unexpectedq",
-                                MPI_T_VERBOSITY_USER_DETAIL,
-                                MPI_T_PVAR_CLASS_TIMER,
-                                MPI_DOUBLE,
-                                MPI_T_ENUM_NULL,
-                                "total time spent on search passes on the unexpected receive queue",
-                                MPI_T_BIND_NO_OBJECT,
-                                /*readonly=*/FALSE,
-                                /*continuous=*/TRUE,
-                                /*atomic=*/FALSE,
-                                MPIR_T_PVAR_IMPL_SIMPLE,
-                                /*var_state=*/&time_matching_unexpectedq,
-                                &simple_double_creator,
-                                &idx);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
-    /* allocated buffer size in the unexpected receive queue */
-    mpi_errno = MPIR_T_pvar_add("unexpected_recvq_buffer_size",
-                                MPI_T_VERBOSITY_USER_DETAIL,
-                                MPI_T_PVAR_CLASS_LEVEL,
-                                MPI_AINT,
-                                MPI_T_ENUM_NULL,
-                                "total buffer size allocated in the unexpected receive queue",
-                                MPI_T_BIND_NO_OBJECT,
-                                /*readonly=*/TRUE,
-                                /*continuous=*/TRUE,
-                                /*atomic=*/FALSE,
-                                MPIR_T_PVAR_IMPL_SIMPLE,
-                                /*var_state=*/&MPIDI_CH3I_unexpected_recvq_buffer_size,
-                                &simple_aint_creator,
-                                &idx);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    MPIR_T_PVAR_LEVEL_REGISTER_STATIC(
+        RECVQ,
+        MPI_UNSIGNED,
+        posted_recvq_length,
+        0, /* init value */
+        MPI_T_VERBOSITY_USER_DETAIL,
+        MPI_T_BIND_NO_OBJECT,
+        MPIR_T_PVAR_FLAG_READONLY | MPIR_T_PVAR_FLAG_CONTINUOUS,
+        "CH3", /* category name */
+        "length of the posted message receive queue");
+
+    MPIR_T_PVAR_LEVEL_REGISTER_STATIC(
+        RECVQ,
+        MPI_UNSIGNED,
+        unexpected_recvq_length,
+        0, /* init value */
+        MPI_T_VERBOSITY_USER_DETAIL,
+        MPI_T_BIND_NO_OBJECT,
+        MPIR_T_PVAR_FLAG_READONLY | MPIR_T_PVAR_FLAG_CONTINUOUS,
+        "CH3", /* category name */
+        "length of the unexpected message receive queue");
+
+    MPIR_T_PVAR_COUNTER_REGISTER_STATIC(
+        RECVQ,
+        MPI_UNSIGNED_LONG_LONG,
+        posted_recvq_match_attempts,
+        MPI_T_VERBOSITY_USER_DETAIL,
+        MPI_T_BIND_NO_OBJECT,
+        MPIR_T_PVAR_FLAG_READONLY | MPIR_T_PVAR_FLAG_CONTINUOUS,
+        "CH3", /* category name */
+        "number of search passes on the message receive queue");
+
+    MPIR_T_PVAR_COUNTER_REGISTER_STATIC(
+        RECVQ,
+        MPI_UNSIGNED_LONG_LONG,
+        unexpected_recvq_match_attempts,
+        MPI_T_VERBOSITY_USER_DETAIL,
+        MPI_T_BIND_NO_OBJECT,
+        MPIR_T_PVAR_FLAG_READONLY | MPIR_T_PVAR_FLAG_CONTINUOUS,
+        "CH3",
+        "number of search passes on the message receive queue");
+
+    MPIR_T_PVAR_TIMER_REGISTER_STATIC(
+        RECVQ,
+        MPI_DOUBLE,
+        time_failed_matching_postedq,
+        MPI_T_VERBOSITY_USER_DETAIL,
+        MPI_T_BIND_NO_OBJECT,
+        MPIR_T_PVAR_FLAG_READONLY | MPIR_T_PVAR_FLAG_CONTINUOUS,
+        "CH3", /* category name */
+        "total time spent on unsuccessful search passes on the posted receives queue");
+
+    MPIR_T_PVAR_TIMER_REGISTER_STATIC(
+        RECVQ,
+        MPI_DOUBLE,
+        time_matching_unexpectedq,
+        MPI_T_VERBOSITY_USER_DETAIL,
+        MPI_T_BIND_NO_OBJECT,
+        MPIR_T_PVAR_FLAG_READONLY | MPIR_T_PVAR_FLAG_CONTINUOUS,
+        "CH3", /* category name */
+        "total time spent on search passes on the unexpected receive queue");
+
+    MPIR_T_PVAR_LEVEL_REGISTER_STATIC(
+        RECVQ,
+        MPI_UNSIGNED_LONG_LONG,
+        unexpected_recvq_buffer_size,
+        0, /* init value */
+        MPI_T_VERBOSITY_USER_DETAIL,
+        MPI_T_BIND_NO_OBJECT,
+        MPIR_T_PVAR_FLAG_READONLY | MPIR_T_PVAR_FLAG_CONTINUOUS,
+        "CH3", /* category name */
+        "total buffer size allocated in the unexpected receive queue");
 
-#endif
 fn_fail:
     return mpi_errno;
 }
@@ -268,7 +214,6 @@ fn_fail:
 int MPIDI_CH3U_Recvq_FU(int source, int tag, int context_id, MPI_Status *s)
 {
     MPID_Request * rreq;
-    MPID_Time_t timer_start;
     int found = 0;
     MPIDI_Message_match match, mask;
     MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FU);
@@ -289,14 +234,14 @@ int MPIDI_CH3U_Recvq_FU(int source, int tag, int context_id, MPI_Status *s)
      * (or it is masked away at some other level). */
     MPIR_TAG_CLEAR_ERROR_BIT(mask.parts.tag);
     if (tag != MPI_ANY_TAG && source != MPI_ANY_SOURCE) {
-        MPIR_T_START_TIMER(RECVQ_STATISTICS, timer_start);
+        MPIR_T_PVAR_TIMER_START(RECVQ, time_matching_unexpectedq);
 	while (rreq != NULL) {
-            MPIR_T_INC(RECVQ_STATISTICS, unexpected_recvq_match_attempts);
+        MPIR_T_PVAR_COUNTER_INC(RECVQ, unexpected_recvq_match_attempts, 1);
 	    if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask))
 		break;
 	    rreq = rreq->dev.next;
 	}
-        MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_matching_unexpectedq);
+        MPIR_T_PVAR_TIMER_END(RECVQ, time_matching_unexpectedq);
     }
     else {
 	if (tag == MPI_ANY_TAG)
@@ -304,14 +249,14 @@ int MPIDI_CH3U_Recvq_FU(int source, int tag, int context_id, MPI_Status *s)
 	if (source == MPI_ANY_SOURCE)
 	    match.parts.rank = mask.parts.rank = 0;
 
-        MPIR_T_START_TIMER(RECVQ_STATISTICS, timer_start);
+        MPIR_T_PVAR_TIMER_START(RECVQ, time_matching_unexpectedq);
 	while (rreq != NULL) {
-            MPIR_T_INC(RECVQ_STATISTICS, unexpected_recvq_match_attempts);
+        MPIR_T_PVAR_COUNTER_INC(RECVQ, unexpected_recvq_match_attempts, 1);
 	    if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask))
 		break;
 	    rreq = rreq->dev.next;
 	}
-        MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_matching_unexpectedq);
+        MPIR_T_PVAR_TIMER_END(RECVQ, time_matching_unexpectedq);
     }
 
     /* Save the information about the request before releasing the 
@@ -350,7 +295,6 @@ int MPIDI_CH3U_Recvq_FU(int source, int tag, int context_id, MPI_Status *s)
 MPID_Request * MPIDI_CH3U_Recvq_FDU(MPI_Request sreq_id, 
 				    MPIDI_Message_match * match)
 {
-    MPID_Time_t timer_start;
     MPID_Request * rreq;
     MPID_Request * prev_rreq;
     MPID_Request * cur_rreq;
@@ -378,17 +322,17 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU(MPI_Request sreq_id,
     /* FIXME: Why doesn't this exit after it finds the first match? */
     cur_rreq = recvq_unexpected_head;
     while (cur_rreq != NULL) {
-        MPIR_T_START_TIMER(RECVQ_STATISTICS, timer_start);
+        MPIR_T_PVAR_TIMER_START(RECVQ, time_matching_unexpectedq);
 
         if (cur_rreq->dev.sender_req_id == sreq_id) {
-            MPIR_T_INC(RECVQ_STATISTICS, unexpected_recvq_match_attempts);
-
+            MPIR_T_PVAR_COUNTER_INC(RECVQ, unexpected_recvq_match_attempts, 1);
             if (MATCH_WITH_LEFT_MASK(cur_rreq->dev.match, *match, mask)) {
                 matching_prev_rreq = prev_rreq;
                 matching_cur_rreq = cur_rreq;
             }
-	}
-        MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_matching_unexpectedq);
+	    }
+
+        MPIR_T_PVAR_TIMER_END(RECVQ, time_matching_unexpectedq);
 
 	prev_rreq = cur_rreq;
 	cur_rreq = cur_rreq->dev.next;
@@ -406,10 +350,10 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU(MPI_Request sreq_id,
 	    recvq_unexpected_tail = matching_prev_rreq;
 	}
 
-        MPIR_T_DEC(RECVQ_STATISTICS, unexpected_qlen);
+    MPIR_T_PVAR_LEVEL_DEC(RECVQ, unexpected_recvq_length, 1);
 	rreq = matching_cur_rreq;
 
-        MPIR_T_SUBTRACT(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
+        MPIR_T_PVAR_LEVEL_DEC(RECVQ, unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
     }
     else {
 	rreq = NULL;
@@ -429,7 +373,6 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU(MPI_Request sreq_id,
 #define FCNAME MPIDI_QUOTE(FUNCNAME)
 MPID_Request * MPIDI_CH3U_Recvq_FDU_matchonly(int source, int tag, int context_id, MPID_Comm *comm, int *foundp)
 {
-    MPID_Time_t timer_start;
     int found = FALSE;
     MPID_Request *rreq, *prev_rreq;
     MPIDI_Message_match match;
@@ -441,7 +384,7 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU_matchonly(int source, int tag, int context_i
     MPIU_THREAD_CS_ASSERT_HELD(MSGQUEUE);
 
     /* Store how much time is spent traversing the queue */
-    MPIR_T_START_TIMER(RECVQ_STATISTICS, timer_start);
+    MPIR_T_PVAR_TIMER_START(RECVQ, time_matching_unexpectedq);
 
     /* Optimize this loop for an empty unexpected receive queue */
     rreq = recvq_unexpected_head;
@@ -460,7 +403,7 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU_matchonly(int source, int tag, int context_i
 
         if (tag != MPI_ANY_TAG && source != MPI_ANY_SOURCE) {
             do {
-                MPIR_T_INC(RECVQ_STATISTICS, unexpected_recvq_match_attempts);
+                MPIR_T_PVAR_COUNTER_INC(RECVQ, unexpected_recvq_match_attempts, 1);
                 if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask)) {
                     if (prev_rreq != NULL) {
                         prev_rreq->dev.next = rreq->dev.next;
@@ -472,8 +415,8 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU_matchonly(int source, int tag, int context_i
                     if (rreq->dev.next == NULL) {
                         recvq_unexpected_tail = prev_rreq;
                     }
-                    MPIR_T_DEC(RECVQ_STATISTICS, unexpected_qlen);
-                    MPIR_T_SUBTRACT(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
+                    MPIR_T_PVAR_LEVEL_DEC(RECVQ, unexpected_recvq_length, 1);
+                    MPIR_T_PVAR_LEVEL_DEC(RECVQ, unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
 
                     rreq->comm = comm;
                     MPIR_Comm_add_ref(comm);
@@ -493,7 +436,7 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU_matchonly(int source, int tag, int context_i
                 match.parts.rank = mask.parts.rank = 0;
 
             do {
-                MPIR_T_INC(RECVQ_STATISTICS, unexpected_recvq_match_attempts);
+                MPIR_T_PVAR_COUNTER_INC(RECVQ, unexpected_recvq_match_attempts, 1);
                 if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask)) {
                     if (prev_rreq != NULL) {
                         prev_rreq->dev.next = rreq->dev.next;
@@ -504,8 +447,8 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU_matchonly(int source, int tag, int context_i
                     if (rreq->dev.next == NULL) {
                         recvq_unexpected_tail = prev_rreq;
                     }
-                    MPIR_T_DEC(RECVQ_STATISTICS, unexpected_qlen);
-                    MPIR_T_SUBTRACT(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
+                    MPIR_T_PVAR_LEVEL_DEC(RECVQ, unexpected_recvq_length, 1);
+                    MPIR_T_PVAR_LEVEL_DEC(RECVQ, unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
 
                     rreq->comm                 = comm;
                     MPIR_Comm_add_ref(comm);
@@ -521,7 +464,7 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU_matchonly(int source, int tag, int context_i
     }
 
 lock_exit:
-    MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_matching_unexpectedq);
+    MPIR_T_PVAR_TIMER_END(RECVQ, time_matching_unexpectedq);
 
     *foundp = found;
 
@@ -551,7 +494,6 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag,
                                            int context_id, MPID_Comm *comm, void *user_buf,
                                            int user_count, MPI_Datatype datatype, int * foundp)
 {
-    MPID_Time_t timer_start;
     int found;
     MPID_Request *rreq, *prev_rreq;
     MPIDI_Message_match match;
@@ -563,7 +505,7 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag,
     MPIU_THREAD_CS_ASSERT_HELD(MSGQUEUE);
 
     /* Store how much time is spent traversing the queue */
-    MPIR_T_START_TIMER(RECVQ_STATISTICS, timer_start);
+    MPIR_T_PVAR_TIMER_START(RECVQ, time_matching_unexpectedq);
 
     /* Optimize this loop for an empty unexpected receive queue */
     rreq = recvq_unexpected_head;
@@ -582,7 +524,7 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag,
 
 	if (tag != MPI_ANY_TAG && source != MPI_ANY_SOURCE) {
 	    do {
-                MPIR_T_INC(RECVQ_STATISTICS, unexpected_recvq_match_attempts);
+            MPIR_T_PVAR_COUNTER_INC(RECVQ, unexpected_recvq_match_attempts, 1);
 		if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask)) {
 		    if (prev_rreq != NULL) {
 			prev_rreq->dev.next = rreq->dev.next;
@@ -594,10 +536,10 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag,
 		    if (rreq->dev.next == NULL) {
 			recvq_unexpected_tail = prev_rreq;
 		    }
-                    MPIR_T_DEC(RECVQ_STATISTICS, unexpected_qlen);
+            MPIR_T_PVAR_LEVEL_DEC(RECVQ, unexpected_recvq_length, 1);
 
-                    if (MPIDI_Request_get_msg_type(rreq) == MPIDI_REQUEST_EAGER_MSG)
-                        MPIR_T_SUBTRACT(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
+            if (MPIDI_Request_get_msg_type(rreq) == MPIDI_REQUEST_EAGER_MSG)
+                MPIR_T_PVAR_LEVEL_DEC(RECVQ, unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
 
 		    rreq->comm = comm;
 		    MPIR_Comm_add_ref(comm);
@@ -618,7 +560,7 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag,
 		match.parts.rank = mask.parts.rank = 0;
 
 	    do {
-                MPIR_T_INC(RECVQ_STATISTICS, unexpected_recvq_match_attempts);
+            MPIR_T_PVAR_COUNTER_INC(RECVQ, unexpected_recvq_match_attempts, 1);
 		if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask)) {
 		    if (prev_rreq != NULL) {
 			prev_rreq->dev.next = rreq->dev.next;
@@ -629,10 +571,10 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag,
 		    if (rreq->dev.next == NULL) {
 			recvq_unexpected_tail = prev_rreq;
 		    }
-                    MPIR_T_DEC(RECVQ_STATISTICS, unexpected_qlen);
+            MPIR_T_PVAR_LEVEL_DEC(RECVQ, unexpected_recvq_length, 1);
 
-                    if (MPIDI_Request_get_msg_type(rreq) == MPIDI_REQUEST_EAGER_MSG)
-                        MPIR_T_SUBTRACT(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
+            if (MPIDI_Request_get_msg_type(rreq) == MPIDI_REQUEST_EAGER_MSG)
+                MPIR_T_PVAR_LEVEL_DEC(RECVQ, unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
 
 		    rreq->comm                 = comm;
 		    MPIR_Comm_add_ref(comm);
@@ -647,7 +589,7 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag,
 	    } while (rreq);
 	}
     }
-    MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_matching_unexpectedq);
+    MPIR_T_PVAR_TIMER_END(RECVQ, time_matching_unexpectedq);
 
     /* A matching request was not found in the unexpected queue, so we 
        need to allocate a new request and add it to the posted queue */
@@ -705,7 +647,7 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag,
 	    recvq_posted_head = rreq;
 	}
 	recvq_posted_tail = rreq;
-        MPIR_T_INC(RECVQ_STATISTICS, posted_qlen);
+    MPIR_T_PVAR_LEVEL_INC(RECVQ, posted_recvq_length, 1);
 	MPIDI_POSTED_RECV_ENQUEUE_HOOK(rreq);
     }
     
@@ -714,7 +656,7 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag,
 
     /* If a match was not found, the timer was stopped after the traversal */
     if (found)
-        MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_matching_unexpectedq);
+        MPIR_T_PVAR_TIMER_END(RECVQ, time_matching_unexpectedq);
     
     MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
     return rreq;
@@ -736,7 +678,6 @@ MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag,
 int MPIDI_CH3U_Recvq_DP(MPID_Request * rreq)
 {
     int found;
-    MPID_Time_t timer_start;
     MPID_Request * cur_rreq;
     MPID_Request * prev_rreq;
     int dequeue_failed;
@@ -749,7 +690,7 @@ int MPIDI_CH3U_Recvq_DP(MPID_Request * rreq)
 
     /* MT FIXME is this right? or should the caller do this? */
     MPIU_THREAD_CS_ENTER(MSGQUEUE,);
-    MPIR_T_START_TIMER(RECVQ_STATISTICS, timer_start);
+    MPIR_T_PVAR_TIMER_START(RECVQ, time_failed_matching_postedq);
     cur_rreq = recvq_posted_head;
     while (cur_rreq != NULL) {
 	if (cur_rreq == rreq) {
@@ -762,7 +703,7 @@ int MPIDI_CH3U_Recvq_DP(MPID_Request * rreq)
 	    if (cur_rreq->dev.next == NULL) {
 		recvq_posted_tail = prev_rreq;
 	    }
-            MPIR_T_DEC(RECVQ_STATISTICS, posted_qlen);
+        MPIR_T_PVAR_LEVEL_DEC(RECVQ, posted_recvq_length, 1);
             /* Notify channel that rreq has been dequeued and check if
                it has already matched rreq, fail if so */
 	    dequeue_failed = MPIDI_POSTED_RECV_DEQUEUE_HOOK(rreq);
@@ -775,7 +716,7 @@ int MPIDI_CH3U_Recvq_DP(MPID_Request * rreq)
 	cur_rreq = cur_rreq->dev.next;
     }
     if (!found)
-        MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_failed_matching_postedq);
+        MPIR_T_PVAR_TIMER_END(RECVQ, time_failed_matching_postedq);
 
     MPIU_THREAD_CS_EXIT(MSGQUEUE,);
 
@@ -809,7 +750,6 @@ int MPIDI_CH3U_Recvq_DP(MPID_Request * rreq)
 MPID_Request * MPIDI_CH3U_Recvq_FDP_or_AEU(MPIDI_Message_match * match, 
 					   int * foundp)
 {
-    MPID_Time_t timer_start;
     int found;
     MPID_Request * rreq;
     MPID_Request * prev_rreq;
@@ -834,9 +774,9 @@ MPID_Request * MPIDI_CH3U_Recvq_FDP_or_AEU(MPIDI_Message_match * match,
 
     rreq = recvq_posted_head;
 
-    MPIR_T_START_TIMER(RECVQ_STATISTICS, timer_start);
+    MPIR_T_PVAR_TIMER_START(RECVQ, time_failed_matching_postedq);
     while (rreq != NULL) {
-        MPIR_T_INC(RECVQ_STATISTICS, posted_recvq_match_attempts);
+        MPIR_T_PVAR_COUNTER_INC(RECVQ, posted_recvq_match_attempts, 1);
 	if (MATCH_WITH_LEFT_RIGHT_MASK(rreq->dev.match, *match, rreq->dev.mask)) {
 	    if (prev_rreq != NULL) {
 		prev_rreq->dev.next = rreq->dev.next;
@@ -847,7 +787,7 @@ MPID_Request * MPIDI_CH3U_Recvq_FDP_or_AEU(MPIDI_Message_match * match,
 	    if (rreq->dev.next == NULL) {
 		recvq_posted_tail = prev_rreq;
 	    }
-            MPIR_T_DEC(RECVQ_STATISTICS, posted_qlen);
+        MPIR_T_PVAR_LEVEL_DEC(RECVQ, posted_recvq_length, 1);
 
             /* give channel a chance to match the request, try again if so */
 	    channel_matched = MPIDI_POSTED_RECV_DEQUEUE_HOOK(rreq);
@@ -860,7 +800,7 @@ MPID_Request * MPIDI_CH3U_Recvq_FDP_or_AEU(MPIDI_Message_match * match,
 	prev_rreq = rreq;
 	rreq = rreq->dev.next;
     }
-    MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_failed_matching_postedq);
+    MPIR_T_PVAR_TIMER_END(RECVQ, time_failed_matching_postedq);
 
     /* A matching request was not found in the posted queue, so we 
        need to allocate a new request and add it to the unexpected queue */
@@ -882,7 +822,7 @@ MPID_Request * MPIDI_CH3U_Recvq_FDP_or_AEU(MPIDI_Message_match * match,
 	    recvq_unexpected_head = rreq;
 	}
 	recvq_unexpected_tail = rreq;
-        MPIR_T_INC(RECVQ_STATISTICS, unexpected_qlen);
+    MPIR_T_PVAR_LEVEL_INC(RECVQ, unexpected_recvq_length, 1);
     }
     
     found = FALSE;
@@ -933,7 +873,7 @@ static inline void dequeue_and_set_error(MPID_Request **req,  MPID_Request *prev
     if (recvq_posted_tail == *req)
         recvq_posted_tail = prev_req;
 
-    MPIR_T_DEC(RECVQ_STATISTICS, posted_qlen);
+    MPIR_T_PVAR_LEVEL_DEC(RECVQ, posted_recvq_length, 1);
 
     /* set error and complete */
     (*req)->status.MPI_ERROR = *error;

http://git.mpich.org/mpich.git/commitdiff/068ac9cd59b6658db9946527ec9d978ebf847279

commit 068ac9cd59b6658db9946527ec9d978ebf847279
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Fri Oct 18 15:13:55 2013 -0500

    Adapt old nemesis pvars to new MPI_T impl
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/src/mpid/ch3/channels/nemesis/include/mpid_nem_inline.h b/src/mpid/ch3/channels/nemesis/include/mpid_nem_inline.h
index 6d215a1..32eb82d 100644
--- a/src/mpid/ch3/channels/nemesis/include/mpid_nem_inline.h
+++ b/src/mpid/ch3/channels/nemesis/include/mpid_nem_inline.h
@@ -12,6 +12,7 @@
 #include "my_papi_defs.h"
 #include "mpiiov.h"
 #include "mpidi_nem_statistics.h"
+#include "mpit.h"
 
 extern int MPID_nem_lmt_shm_pending;
 extern MPID_nem_cell_ptr_t MPID_nem_prefetched_cell;
@@ -83,7 +84,7 @@ MPID_nem_mpich_send_header (void* buf, int size, MPIDI_VC_t *vc, int *again)
         goto return_success;
     }
  usequeue_l:
-    MPIR_T_INC(NEM_STATISTICS, MPID_nem_fbox_fall_back_to_queue_count[MPID_nem_mem_region.local_ranks[vc->lpid]]);
+    MPIR_T_PVAR_COUNTER_INC_VAR(NEM, &MPID_nem_fbox_fall_back_to_queue_count[MPID_nem_mem_region.local_ranks[vc->lpid]], 1);
 
 #endif /*USE_FASTBOX */
 
@@ -315,7 +316,7 @@ MPID_nem_mpich_sendv_header (MPID_IOV **iov, int *n_iov, MPIDI_VC_t *vc, int *ag
         goto return_success;
     }
  usequeue_l:
-    MPIR_T_INC(NEM_STATISTICS, MPID_nem_fbox_fall_back_to_queue_count[MPID_nem_mem_region.local_ranks[vc->lpid]]);
+    MPIR_T_PVAR_COUNTER_INC_VAR(NEM, &MPID_nem_fbox_fall_back_to_queue_count[MPID_nem_mem_region.local_ranks[vc->lpid]], 1);
 
 #endif /*USE_FASTBOX */
 	
@@ -469,7 +470,7 @@ MPID_nem_mpich_send_seg_header (MPID_Segment *segment, MPIDI_msg_sz_t *segment_f
 	}
     }
  usequeue_l:
-    MPIR_T_INC(NEM_STATISTICS, MPID_nem_fbox_fall_back_to_queue_count[MPID_nem_mem_region.local_ranks[vc->lpid]]);
+    MPIR_T_PVAR_COUNTER_INC_VAR(NEM, &MPID_nem_fbox_fall_back_to_queue_count[MPID_nem_mem_region.local_ranks[vc->lpid]], 1);
 
 #endif /*USE_FASTBOX */
 	
diff --git a/src/mpid/ch3/channels/nemesis/include/mpidi_nem_statistics.h b/src/mpid/ch3/channels/nemesis/include/mpidi_nem_statistics.h
index bdf6a0a..1afbb0d 100644
--- a/src/mpid/ch3/channels/nemesis/include/mpidi_nem_statistics.h
+++ b/src/mpid/ch3/channels/nemesis/include/mpidi_nem_statistics.h
@@ -7,9 +7,6 @@
 #ifndef _MPIDI_NEM_STATISTICS_H_
 #define _MPIDI_NEM_STATISTICS_H_
 
-#include "mpidi_common_statistics.h"
-
-/* Do not protect by enable macro - see mpidi_common_statistics.h */
-extern uint64_t *MPID_nem_fbox_fall_back_to_queue_count;
+extern unsigned long long *MPID_nem_fbox_fall_back_to_queue_count;
 
 #endif  /* _MPIDI_NEM_STATISTICS_H_ */
diff --git a/src/mpid/ch3/channels/nemesis/src/ch3i_comm.c b/src/mpid/ch3/channels/nemesis/src/ch3i_comm.c
index f25c9a7..f95c25c 100644
--- a/src/mpid/ch3/channels/nemesis/src/ch3i_comm.c
+++ b/src/mpid/ch3/channels/nemesis/src/ch3i_comm.c
@@ -5,6 +5,7 @@
  */
 
 #include "mpid_nem_impl.h"
+#undef utarray_oom
 #define utarray_oom() do { goto fn_oom; } while (0)
 #include "mpiu_utarray.h"
 
diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_finalize.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_finalize.c
index f5cc1a5..a5c636a 100644
--- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_finalize.c
+++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_finalize.c
@@ -56,7 +56,7 @@ int MPID_nem_finalize(void)
     my_papi_close();
 #endif /*PAPI_MONITOR */
     
-#if ENABLE_NEM_STATISTICS
+#if ENABLE_PVAR_NEM
     MPIU_Free(MPID_nem_fbox_fall_back_to_queue_count);
 #endif
 
diff --git a/src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c b/src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c
index a928362..80e1556 100644
--- a/src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c
+++ b/src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c
@@ -9,6 +9,7 @@
 #include "mpid_nem_nets.h"
 #include <errno.h>
 #include "mpidi_nem_statistics.h"
+#include "mpit.h"
 
 /* constants for configure time selection of local LMT implementations */
 #define MPID_NEM_LOCAL_LMT_NONE 0
@@ -38,11 +39,7 @@ static int get_local_procs(MPIDI_PG_t *pg, int our_pg_rank, int *num_local_p,
 char *MPID_nem_asymm_base_addr = 0;
 
 /* used by mpid_nem_inline.h and mpid_nem_finalize.c */
-uint64_t *MPID_nem_fbox_fall_back_to_queue_count = NULL;
-
-#if ENABLE_NEM_STATISTICS
-/* MPIT support */
-MPIR_T_SIMPLE_HANDLE_CREATOR(fbox_count_creator, uint64_t, MPID_nem_mem_region.num_local)
+unsigned long long *MPID_nem_fbox_fall_back_to_queue_count = NULL;
 
 #undef FUNCNAME
 #define FUNCNAME MPID_nem_init_stats
@@ -51,32 +48,30 @@ MPIR_T_SIMPLE_HANDLE_CREATOR(fbox_count_creator, uint64_t, MPID_nem_mem_region.n
 static int MPID_nem_init_stats(int n_local_ranks)
 {
     int mpi_errno = MPI_SUCCESS;
-    int idx = -1;
-
-    MPID_nem_fbox_fall_back_to_queue_count = MPIU_Calloc(n_local_ranks, sizeof(uint64_t));
-
-    mpi_errno = MPIR_T_pvar_add("nem_fbox_fall_back_to_queue_count",
-                                MPI_T_VERBOSITY_USER_DETAIL,
-                                MPI_T_PVAR_CLASS_COUNTER,
-                                MPI_AINT,
-                                MPI_T_ENUM_NULL,
-                                "array counting how many times nemesis had to fall back to "
-                                "the regular queue when sending messages between pairs of "
-                                "local processes",
-                                MPI_T_BIND_NO_OBJECT,
-                                /*readonly=*/ FALSE,
-                                /*continuous=*/ TRUE,
-                                /*atomic=*/ FALSE,
-                                MPIR_T_PVAR_IMPL_SIMPLE,
-                                /*var_state=*/ MPID_nem_fbox_fall_back_to_queue_count,
-                                &fbox_count_creator,
-                                &idx);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
 
-fn_fail:
+#ifdef ENABLE_PVAR_NEM
+    MPID_nem_fbox_fall_back_to_queue_count = MPIU_Calloc(n_local_ranks, sizeof(unsigned long long));
+#endif
+
+    MPIR_T_PVAR_COUNTER_REGISTER_DYNAMIC(
+        NEM,
+        MPI_UNSIGNED_LONG_LONG,
+        nem_fbox_fall_back_to_queue_count, /* name */
+        MPID_nem_fbox_fall_back_to_queue_count, /* address */
+        n_local_ranks, /* count, known at pvar registeration time */
+        MPI_T_VERBOSITY_USER_DETAIL,
+        MPI_T_BIND_NO_OBJECT,
+        MPIR_T_PVAR_FLAG_CONTINUOUS, /* flags */
+        NULL, /* get_value */
+        NULL, /* get_count */
+        "NEMESIS", /* category */
+        "Array counting how many times nemesis had to fall back to the regular queue when sending messages between pairs of local processes");
+
+fn_exit:
     return mpi_errno;
+fn_fail:
+    goto fn_exit;
 }
-#endif  /* ENABLE_NEM_STATISTICS */
 
 #undef FUNCNAME
 #define FUNCNAME MPID_nem_init
@@ -383,9 +378,7 @@ MPID_nem_init(int pg_rank, MPIDI_PG_t *pg_p, int has_parent ATTRIBUTE((unused)))
     my_papi_start( pg_rank );
 #endif /*PAPI_MONITOR   */
 
-#if ENABLE_NEM_STATISTICS
     MPID_nem_init_stats(num_local);
-#endif
 
     MPIU_CHKPMEM_COMMIT();
  fn_exit:

http://git.mpich.org/mpich.git/commitdiff/b351a97e86367f2c21b59ee1a55e835a820d1374

commit b351a97e86367f2c21b59ee1a55e835a820d1374
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Fri Oct 18 14:42:27 2013 -0500

    Change wording for option --enable-mpit-pvars
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/configure.ac b/configure.ac
index 9271973..2820478 100644
--- a/configure.ac
+++ b/configure.ac
@@ -376,9 +376,9 @@ AC_ARG_ENABLE(g,
 ],,enable_g=none)
 
 AC_ARG_ENABLE([mpit_pvars],
-[--enable-mpit-pvars=list - Selectively enable MPI_T performance variables.
-                         list is a comma-separated list of variable names,
-                         including
+[  --enable-mpit-pvars=list - Selectively enable MPI_T performance variables in
+                      modules. list is a comma-separated module names,
+                      including (Default is "all"):
         none     - No performance info recorded
         recvq    - All message queue-related
         nem      - All nemesis-related

http://git.mpich.org/mpich.git/commitdiff/15357039dbebe52910cf3afdc251f8deaadcf38d

commit 15357039dbebe52910cf3afdc251f8deaadcf38d
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Fri Oct 18 14:27:15 2013 -0500

    Change var names in enable_pvar_* style
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/configure.ac b/configure.ac
index e50d434..9271973 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1842,11 +1842,11 @@ save_IFS="$IFS"
 IFS=","
 for var in $enable_mpit_pvars ; do
     AS_CASE(["$var"],
-            [nem],[enable_nem_pvars=yes],
-            [recvq],[enable_recvq_pvars=yes],
+            [nem],[enable_pvar_nem=yes],
+            [recvq],[enable_pvar_recvq=yes],
             [all|yes],
-            [enable_nem_pvars=yes
-             enable_recvq_pvars=yes],
+            [enable_pvar_nem=yes
+             enable_pvar_recvq=yes],
             [no|none],[],
             [IFS=$save_IFS
              AC_MSG_WARN([Unknown value ($option) for enable-mpit-pvars])
@@ -1854,20 +1854,20 @@ for var in $enable_mpit_pvars ; do
 done
 IFS="$save_IFS"
 
-if test -n "$enable_nem_pvars" ; then
+if test -n "$enable_pvar_nem" ; then
     status_nem_pvars=1
 else
     status_nem_pvars=0
 fi
-AC_DEFINE_UNQUOTED(ENABLE_NEM_STATISTICS,$status_nem_pvars,
+AC_DEFINE_UNQUOTED(ENABLE_PVAR_NEM,$status_nem_pvars,
           [Define to 1 to enable nemesis-related MPI_T performance variables])
 
-if test -n "$enable_recvq_pvars" ; then
+if test -n "$enable_pvar_recvq" ; then
     status_recvq_pvars=1
 else
     status_recvq_pvars=0
 fi
-AC_DEFINE_UNQUOTED(ENABLE_RECVQ_STATISTICS,$status_recvq_pvars,
+AC_DEFINE_UNQUOTED(ENABLE_PVAR_RECVQ,$status_recvq_pvars,
           [Define to 1 to enable message receive queue-related MPI_T performance variables])
 
 # ---------------------------------------------------------------------------

http://git.mpich.org/mpich.git/commitdiff/bfca138203fae7f64b3b1aeb9e2e1a0945a1b5aa

commit bfca138203fae7f64b3b1aeb9e2e1a0945a1b5aa
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Fri Oct 18 15:00:58 2013 -0500

    Delete the old interfaces on pvars
    
    New code should use interfaces provided by new MPI_T impl.
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/src/mpid/ch3/include/mpidi_common_statistics.h b/src/mpid/ch3/include/mpidi_common_statistics.h
deleted file mode 100644
index d430bca..0000000
--- a/src/mpid/ch3/include/mpidi_common_statistics.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/* -*- Mode: c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2013 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-#ifndef _MPIDI_COMMON_STATISTICS_H_
-#define _MPIDI_COMMON_STATISTICS_H_
-
-/* Statically decides whether or not to perform 'action'.
- *
- * This has the unfortunate side-effect that the compiler will complain about
- * undeclared variables in the 'action' parameter.
- * Thankfully, since by linking time they will be long gone (due to the compiler
- * optimizing away the whole macro), a simple 'extern' declaration suffices.
- * This is why some statistics headers declare their variables as extern
- * outside of the macro scope.
- */
-#define MPIR_T_GATE(PVAR_CLASS, action) \
-    do {                                \
-        if (ENABLE_##PVAR_CLASS) {      \
-            action;                     \
-        }                               \
-    } while(0)
-
-
-/* TODO add some code here and probably elsewhere to make these show up in the
- * MPI_T_pvar_ interface */
-#define MPIR_T_INC_impl(x) (++(x))
-#define MPIR_T_DEC_impl(x) (--(x))
-
-#define MPIR_T_INC(PVAR_CLASS, x) MPIR_T_GATE(PVAR_CLASS, MPIR_T_INC_impl(x))
-#define MPIR_T_DEC(PVAR_CLASS, x) MPIR_T_GATE(PVAR_CLASS, MPIR_T_DEC_impl(x))
-
-
-#define MPIR_T_START_TIMER_impl(start) MPID_Wtime(&start)
-#define MPIR_T_END_TIMER_impl(start, mpit_variable)     \
-    do {                                                \
-        MPID_Time_t end;                                \
-        double temp_delta = 0.0;                        \
-        MPID_Wtime(&(end));                             \
-        MPID_Wtime_diff(&(start), &end, &temp_delta);   \
-        (mpit_variable) += temp_delta;                  \
-    } while(0)
-
-#define MPIR_T_START_TIMER(PVAR_CLASS, start) MPIR_T_GATE(PVAR_CLASS, MPIR_T_START_TIMER_impl(start))
-#define MPIR_T_END_TIMER(PVAR_CLASS, start, pvar) MPIR_T_GATE(PVAR_CLASS, MPIR_T_END_TIMER_impl(start, pvar))
-
-
-#define MPIR_T_SUBTRACT_impl(x, y) ((x) -= (y))
-#define MPIR_T_ADD_impl(x, y)      ((x) += (y))
-
-#define MPIR_T_SUBTRACT(PVAR_CLASS, x, y) MPIR_T_GATE(PVAR_CLASS, MPIR_T_SUBTRACT_impl(x, y))
-#define MPIR_T_ADD(PVAR_CLASS, x, y) MPIR_T_GATE(PVAR_CLASS, MPIR_T_ADD_impl(x, y))
-
-
-#define MPIR_T_SIMPLE_HANDLE_CREATOR(TAG, TYPE, COUNT)                          \
-static int TAG(void *obj_handle,                                                \
-               struct MPIR_T_pvar_handle *handle,                               \
-               int *countp)                                                     \
-{                                                                               \
-    /* the IMPL_SIMPLE code reads/writes "bytes" bytes from the location given  \
-     * by the "handle_state" pointer */                                         \
-    handle->handle_state = handle->info->var_state;                             \
-    handle->bytes = sizeof(TYPE);                                               \
-                                                                                \
-    *countp = (COUNT);                                                          \
-    return MPI_SUCCESS;                                                         \
-}
-
-#endif  /* _MPIDI_COMMON_STATISTICS_H_ */
diff --git a/src/mpid/ch3/include/mpidi_recvq_statistics.h b/src/mpid/ch3/include/mpidi_recvq_statistics.h
index d0109a2..9c3f263 100644
--- a/src/mpid/ch3/include/mpidi_recvq_statistics.h
+++ b/src/mpid/ch3/include/mpidi_recvq_statistics.h
@@ -7,10 +7,9 @@
 #ifndef _MPIDI_RECVQ_STATISTICS_H_
 #define _MPIDI_RECVQ_STATISTICS_H_
 
-#include "mpidi_common_statistics.h"
+#include "mpit.h"
 
-/* Do not protect by enable macro - see mpidi_common_statistics.h */
 /* from ch3u_recvq.c */
-extern uint64_t MPIDI_CH3I_unexpected_recvq_buffer_size;
+MPIR_T_PVAR_ULONG2_LEVEL_DECL_EXTERN(RECVQ, unexpected_recvq_buffer_size);
 
 #endif  /* _MPIDI_RECVQ_STATISTICS_H_ */

http://git.mpich.org/mpich.git/commitdiff/21d84da991d484f7924935591416e77845331662

commit 21d84da991d484f7924935591416e77845331662
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Fri Oct 18 15:46:56 2013 -0500

    In MPI init/finalize, use updated MPI_T routines
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/src/mpi/init/finalize.c b/src/mpi/init/finalize.c
index c50e4a7..91e4c47 100644
--- a/src/mpi/init/finalize.c
+++ b/src/mpi/init/finalize.c
@@ -220,21 +220,17 @@ int MPI_Finalize( void )
        completing the finalize */
     if (mpi_errno != MPI_SUCCESS) goto fn_fail;
 
+    /* Users did not call MPI_T_init_thread(), so we free memories allocated to
+     * MPIR_T during MPI_Init here. Otherwise, free them in MPI_T_finalize() */
+    if (!MPIR_T_is_initialized())
+        MPIR_T_env_finalize();
+
     /* FIXME: Many of these debugging items could/should be callbacks, 
        added to the finalize callback list */
     /* FIXME: the memory tracing code block should be a finalize callback */
     /* If memory debugging is enabled, check the memory here, after all
        finalize callbacks */
 
-    /* FIXME The init/finalize paths in general need a big overhaul in order
-     * to account for the new MPI_T_ code. */
-    if (!MPIR_T_is_initialized()) {
-        MPIR_T_finalize_pvars();
-
-        mpi_errno = MPIR_Param_finalize();
-        if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-    }
-
     MPIU_THREAD_CS_EXIT(ALLFUNC,);
     MPIR_Process.initialized = MPICH_POST_FINALIZED;
 
diff --git a/src/mpi/init/init.c b/src/mpi/init/init.c
index b68318a..3d7789d 100644
--- a/src/mpi/init/init.c
+++ b/src/mpi/init/init.c
@@ -101,8 +101,7 @@ int MPI_Init( int *argc, char ***argv )
 
     /* ... body of routine ... */
 
-    mpi_errno = MPIR_Param_init_params();
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    MPIR_T_env_init();
 
     if (!strcmp(MPIR_PARAM_DEFAULT_THREAD_LEVEL, "MPI_THREAD_MULTIPLE"))
         threadLevel = MPI_THREAD_MULTIPLE;
diff --git a/src/mpi/init/initthread.c b/src/mpi/init/initthread.c
index 44a5cfe..b15c66c 100644
--- a/src/mpi/init/initthread.c
+++ b/src/mpi/init/initthread.c
@@ -582,8 +582,7 @@ int MPI_Init_thread( int *argc, char ***argv, int required, int *provided )
 
     /* ... body of routine ... */
 
-    mpi_errno = MPIR_Param_init_params();
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    MPIR_T_env_init();
 
     /* If the user requested for asynchronous progress, request for
      * THREAD_MULTIPLE. */

http://git.mpich.org/mpich.git/commitdiff/72bf15fbd0578876ea731ce8f5630b19d7af3132

commit 72bf15fbd0578876ea731ce8f5630b19d7af3132
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Fri Oct 18 15:39:27 2013 -0500

    Replace mpi_t_util.c with mpit.c
    
    mpit.c also includes non-util routines
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/src/mpi_t/Makefile.mk b/src/mpi_t/Makefile.mk
index f26c398..98d5404 100644
--- a/src/mpi_t/Makefile.mk
+++ b/src/mpi_t/Makefile.mk
@@ -36,5 +36,5 @@ mpi_sources +=                \
         src/mpi_t/pvar_write.c
 
 
-lib_lib at MPILIBNAME@_la_SOURCES += src/mpi_t/mpi_t_util.c
+lib_lib at MPILIBNAME@_la_SOURCES += src/mpi_t/mpit.c
 
diff --git a/src/mpi_t/mpi_t_util.c b/src/mpi_t/mpi_t_util.c
deleted file mode 100644
index 8c28b2e..0000000
--- a/src/mpi_t/mpi_t_util.c
+++ /dev/null
@@ -1,151 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2011 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-/* this file contains helper routines for the implementation of the MPI_T
- * interface */
-
-#include "mpiimpl.h"
-#include "mpiu_utarray.h"
-
-/* A dynamic array of pointers to all known performance variables.  The array
- * will grow dynamically as variables are added by lower level initialization
- * code. */
-static UT_array *all_pvars = NULL;
-
-/* Called by lower-level initialization code to add pvars to the global list.
- * Will cause the value returned by MPI_T_pvar_get_num to be incremented and
- * sets up that new index to work with get_info, handle_alloc, etc. */
-int MPIR_T_pvar_add(const char *name,
-                    enum MPIR_T_verbosity_t verbosity,
-                    enum MPIR_T_pvar_class_t varclass,
-                    MPI_Datatype dtype,
-                    struct MPIR_T_enum *enumtype,
-                    const char *desc,
-                    enum MPIR_T_bind_t binding,
-                    int readonly,
-                    int continuous,
-                    int atomic,
-                    enum MPIR_T_pvar_impl_kind impl_kind,
-                    void *var_state,
-                    MPIR_T_pvar_handle_creator_fn *create_fn,
-                    int *index_p)
-{
-    struct MPIR_T_pvar_info *info;
-
-    if (!all_pvars) {
-        utarray_new(all_pvars, &ut_ptr_icd);
-    }
-
-    info = MPIU_Malloc(sizeof(*info));
-    MPIU_Assert(info != NULL);
-
-    info->name = MPIU_Strdup((name ? name : ""));
-    MPIU_Assert(info->name);
-    info->verbosity = verbosity;
-    info->varclass = varclass;
-    info->dtype = dtype;
-    info->etype = enumtype;
-    info->desc = MPIU_Strdup((desc ? desc : ""));
-    MPIU_Assert(info->desc);
-    info->binding = binding;
-    info->readonly = readonly;
-    info->continuous = continuous;
-    info->atomic = atomic;
-    info->impl_kind = impl_kind;
-    info->var_state = var_state;
-    info->create_fn = create_fn;
-
-    utarray_push_back(all_pvars, &info);
-    *index_p = utarray_len(all_pvars);
-
-    return MPI_SUCCESS;
-}
-
-int MPIR_T_get_num_pvars(int *num)
-{
-    MPIU_Assert(num != NULL);
-    if (all_pvars) {
-        *num = utarray_len(all_pvars);
-    }
-    else {
-        *num = 0;
-    }
-    return MPI_SUCCESS;
-}
-
-int MPIR_T_get_pvar_info_by_idx(int idx, struct MPIR_T_pvar_info **info_p)
-{
-    MPIU_Assert(idx >= 0);
-    MPIU_Assert(all_pvars != NULL);
-    MPIU_Assert(idx < utarray_len(all_pvars));
-
-    *info_p = *(struct MPIR_T_pvar_info **)utarray_eltptr(all_pvars, idx);
-    return MPI_SUCCESS;
-}
-
-int MPIR_T_finalize_pvars(void)
-{
-    struct MPIR_T_pvar_info **info_p;
-    if (all_pvars) {
-        for (info_p = (struct MPIR_T_pvar_info **)utarray_front(all_pvars);
-             info_p != NULL;
-             info_p = (struct MPIR_T_pvar_info **)utarray_next(all_pvars, info_p))
-        {
-            MPIU_Free((*info_p)->name);
-            MPIU_Free((*info_p)->desc);
-            MPIU_Free((*info_p));
-        }
-        utarray_free(all_pvars);
-        all_pvars = NULL;
-    }
-
-    return MPI_SUCCESS;
-}
-
-/* Implements an MPI_T-style strncpy.  Here is the description from the draft
- * standard:
- *
- *   Several MPI tool information interface functions return one or more
- *   strings. These functions have two arguments for each string to be returned:
- *   an OUT parameter that identifies a pointer to the buffer in which the
- *   string will be returned, and an IN/OUT parameter to pass the length of the
- *   buffer. The user is responsible for the memory allocation of the buffer and
- *   must pass the size of the buffer (n) as the length argument. Let n be the
- *   length value specified to the function. On return, the function writes at
- *   most n − 1 of the string’s characters into the buffer, followed by a null
- *   terminator. If the returned string’s length is greater than or equal to n,
- *   the string will be truncated to n − 1 characters. In this case, the length
- *   of the string plus one (for the terminating null character) is returned in
- *   the length argument. If the user passes the null pointer as the buffer
- *   argument or passes 0 as the length argument, the function does not return
- *   the string and only returns the length of the string plus one in the length
- *   argument. If the user passes the null pointer as the length argument, the
- *   buffer argument is ignored and nothing is returned.
- *
- * So this routine copies up to (*len)-1 characters from src to dst and then
- * sets *len to (strlen(dst)+1).  If dst==NULL, just return (strlen(src)+1) in
- * *len.
- *
- * This routine does not follow MPICH error handling conventions.
- */
-void MPIU_Tool_strncpy(char *dst, const char *src, int *len)
-{
-    /* std. says to ignore str arg if len arg is NULL (MPI-3, p.563) */
-    if (len) {
-        MPIU_Assert(*len >= 0);
-
-        if (!dst || !*len) {
-            /* just return the space needed to hold src, including the terminator */
-            *len = strlen(src) + 1;
-        }
-        else {
-            /* MPL_strncpy will always terminate the string */
-            MPL_strncpy(dst, src, *len);
-            *len = strlen(dst) + 1;
-        }
-    }
-}
-
diff --git a/src/mpi_t/mpit.c b/src/mpi_t/mpit.c
new file mode 100644
index 0000000..9ca5f0f
--- /dev/null
+++ b/src/mpi_t/mpit.c
@@ -0,0 +1,467 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
+/*
+ *  (C) 2013 by Argonne National Laboratory.
+ *      See COPYRIGHT in top-level directory.
+ */
+
+#include "mpiimpl.h"
+
+int MPIR_T_init_balance = 0;
+
+#ifdef MPICH_IS_THREADED
+MPIU_Thread_mutex_t mpi_t_mutex;
+#endif
+
+#ifdef HAVE_RUNTIME_THREADCHECK
+int MPIR_T_is_threaded;
+#endif
+
+/* These variables must be initialized in MPI_T_initthread. Especially,
+ * hash table pointers must be initialized to NULL.
+ */
+int cat_stamp;
+UT_array *enum_table;
+UT_array *cat_table;
+UT_array *cvar_table;
+UT_array *pvar_table;
+name2index_hash_t *cat_hash;
+name2index_hash_t *cvar_hash;
+name2index_hash_t *pvar_hashs[MPIR_T_PVAR_CLASS_NUMBER];
+
+/* Create an enum.
+ * IN: enum_name, name of the enum
+ * OUT: handle, handle of the enum
+ */
+void MPIR_T_enum_create(const char *enum_name, MPI_T_enum *handle)
+{
+    MPIR_T_enum_t *e;
+    static const UT_icd enum_item_icd = {sizeof(enum_item_t), NULL, NULL, NULL};
+
+    MPIU_Assert(enum_name);
+    MPIU_Assert(handle);
+
+    utarray_extend_back(enum_table);
+    e = (MPIR_T_enum_t *)utarray_back(enum_table);
+    e->name = MPIU_Strdup(enum_name);
+    MPIU_Assert(e->name);
+    utarray_new(e->items, &enum_item_icd);
+    (*handle) = e;
+}
+
+/* Add an item to an exisiting enum.
+ * IN: handle, handle to the enum
+ * IN: item_name, name of the item
+ * IN: item_value, value associated with item_name
+ */
+void MPIR_T_enum_add_item(MPI_T_enum handle, const char *item_name, int item_value)
+{
+    enum_item_t *item;
+
+    MPIU_Assert(handle);
+    MPIU_Assert(item_name);
+
+    utarray_extend_back(handle->items);
+    item = (enum_item_t *)utarray_back(handle->items);
+    item->name = MPIU_Strdup(item_name);
+    MPIU_Assert(item->name);
+    item->value = item_value;
+}
+
+/* Create a new category with name <cat_name>.
+ * The new category is pushed at the back of cat_table.
+ * Aslo, a new hash entry is added for the category in cat_hash.
+ * Return the newly created category.
+ */
+static cat_table_entry_t *MPIR_T_cat_create(const char *cat_name)
+{
+    int cat_idx;
+    cat_table_entry_t *cat;
+    name2index_hash_t *hash_entry;
+
+    /* New a category */
+    utarray_extend_back(cat_table);
+    cat =(cat_table_entry_t *)utarray_back(cat_table);
+    cat->name = MPIU_Strdup(cat_name);
+    cat->desc = NULL;
+    utarray_new(cat->cvar_indices, &ut_int_icd);
+    utarray_new(cat->pvar_indices, &ut_int_icd);
+    utarray_new(cat->subcat_indices, &ut_int_icd);
+
+    /* Record <cat_name, cat_idx> in cat_hash */
+    cat_idx = utarray_len(cat_table) - 1;
+    hash_entry = MPIU_Malloc(sizeof(name2index_hash_t));
+    MPIU_Assert(hash_entry);
+    /* Need not to Strdup cat_name, since cat_table and cat_hash co-exist */
+    hash_entry->name = cat_name;
+    hash_entry->idx = cat_idx;
+    HASH_ADD_KEYPTR(hh, cat_hash, hash_entry->name,
+                    strlen(hash_entry->name), hash_entry);
+
+    return cat;
+}
+
+/* Add a pvar to an existing or new category
+ * IN: cat_name, name of the category
+ * IN: pvar_index, index of the pvar as defined by MPI_T_pvar_handle_alloc()
+ * If cat_name is NULL or a empty string, nothing happpens.
+ */
+int MPIR_T_cat_add_pvar(const char *cat_name, int pvar_index)
+{
+    int mpi_errno = MPI_SUCCESS;
+    name2index_hash_t *hash_entry;
+    cat_table_entry_t *cat;
+
+    /* NULL or empty string are allowed */
+    if (cat_name == NULL || *cat_name == '\0')
+        goto fn_exit;
+
+    HASH_FIND_STR(cat_hash, cat_name, hash_entry);
+
+    if (hash_entry != NULL) {
+        /* Found it, i.e., category already exists */
+        int cat_idx = hash_entry->idx;
+        cat = (cat_table_entry_t *)utarray_eltptr(cat_table, cat_idx);
+        /* FIXME: Is it worth checking duplicated vars? Probably not */
+        utarray_push_back(cat->pvar_indices, &pvar_index);
+    } else {
+        /* Not found, so create a new category */
+        cat = MPIR_T_cat_create(cat_name);
+        utarray_push_back(cat->pvar_indices, &pvar_index);
+        /* Notify categories have been changed */
+        cat_stamp++;
+    }
+
+fn_exit:
+    return mpi_errno;
+
+fn_fail:
+    goto fn_exit;
+}
+
+/* Add a cvar to an existing or new category
+ * IN: cat_name, name of the category
+ * IN: cvar_index, index of the cvar as defined by MPI_T_cvar_handle_alloc()
+ * If cat_name is NULL or a empty string, nothing happpens.
+ */
+int MPIR_T_cat_add_cvar(const char *cat_name, int cvar_index)
+{
+    int mpi_errno = MPI_SUCCESS;
+    name2index_hash_t *hash_entry;
+    cat_table_entry_t *cat;
+
+    /* NULL or empty string are allowed */
+    if (cat_name == NULL || *cat_name == '\0')
+        goto fn_exit;
+
+    HASH_FIND_STR(cat_hash, cat_name, hash_entry);
+
+    if (hash_entry != NULL) {
+        /* Found it, i.e., category already exists */
+        int cat_idx = hash_entry->idx;
+        cat = (cat_table_entry_t *)utarray_eltptr(cat_table, cat_idx);
+        /* FIXME: Is it worth checking duplicated vars? Probably not */
+        utarray_push_back(cat->cvar_indices, &cvar_index);
+    } else {
+        /* Not found, so create a new category */
+        cat = MPIR_T_cat_create(cat_name);
+        utarray_push_back(cat->cvar_indices, &cvar_index);
+        /* Notify categories have been changed */
+        cat_stamp++;
+    }
+
+fn_exit:
+    return mpi_errno;
+
+fn_fail:
+    goto fn_exit;
+}
+
+/* Add a sub-category to an existing or new category
+ * IN: parent_name, name of the parent category
+ * IN: child_name, name of the child category
+ */
+int MPIR_T_cat_add_subcat(const char *parent_name, const char *child_name)
+{
+    int mpi_errno = MPI_SUCCESS;
+    int parent_index, child_index;
+    name2index_hash_t *hash_entry;
+    cat_table_entry_t *parent;
+
+    /* NULL or empty string are allowed */
+    if (parent_name == NULL || *parent_name == '\0' ||
+        child_name == NULL || *child_name == '\0')
+    {
+        goto fn_exit;
+    }
+
+    /* Find or create parent */
+    HASH_FIND_STR(cat_hash, parent_name, hash_entry);
+    if (hash_entry != NULL) {
+        /* Found parent in cat_table */
+        parent_index = hash_entry->idx;
+    } else {
+        /* parent is a new category */
+        MPIR_T_cat_create(parent_name);
+        parent_index = utarray_len(cat_table) - 1;
+    }
+
+    /* Find or create child */
+    HASH_FIND_STR(cat_hash, child_name, hash_entry);
+    if (hash_entry != NULL) {
+        /* Found child in cat_table */
+        child_index = hash_entry->idx;
+    } else {
+        /* child is a new category */
+        MPIR_T_cat_create(child_name);
+        child_index = utarray_len(cat_table) - 1;
+    }
+
+    /* Connect parent and child */
+    parent = (cat_table_entry_t *)utarray_eltptr(cat_table, parent_index);
+    utarray_push_back(parent->subcat_indices, &child_index);
+
+    /* Notify categories have been changed */
+    cat_stamp++;
+
+fn_exit:
+    return mpi_errno;
+
+fn_fail:
+    goto fn_exit;
+}
+
+/* Add description to an existing or new category
+ * IN: cat_name, name of the category
+ * IN: cat_desc, description of the category
+ */
+int MPIR_T_cat_add_desc(const char *cat_name, const char *cat_desc)
+{
+    int cat_idx, mpi_errno = MPI_SUCCESS;
+    name2index_hash_t *hash_entry;
+    cat_table_entry_t *cat;
+
+    /* NULL args are not allowed */
+    MPIU_Assert(cat_name);
+    MPIU_Assert(cat_desc);
+
+    HASH_FIND_STR(cat_hash, cat_name, hash_entry);
+
+    if (hash_entry != NULL) {
+        /* Found it, i.e., category already exists */
+        cat_idx = hash_entry->idx;
+        cat = (cat_table_entry_t *)utarray_eltptr(cat_table, cat_idx);
+        MPIU_Assert(cat->desc == NULL);
+        cat->desc = MPIU_Strdup(cat_desc);
+        MPIU_Assert(cat->desc);
+    } else {
+        /* Not found, so create a new category */
+        cat = MPIR_T_cat_create(cat_name);
+        cat->desc = MPIU_Strdup(cat_desc);
+        MPIU_Assert(cat->desc);
+        /* Notify categories have been changed */
+        cat_stamp++;
+    }
+
+fn_exit:
+    return mpi_errno;
+
+fn_fail:
+    goto fn_exit;
+}
+
+/* A low level, generic and internally used interface to register
+ * a cvar to the MPIR_T.
+ *
+ * IN: dtype, MPI datatype for this cvar
+ * IN: name, Name of the cvar
+ * IN: addr, Pointer to the cvar if known at registeration, otherwise NULL.
+ * IN: count, # of elements of this cvar if known at registeration, otherwise 0.
+ * IN: etype, MPI_T_enum or MPI_T_ENUM_NULL
+ * IN: verb, MPI_T_PVAR_VERBOSITY_*
+ * IN: bind, MPI_T_BIND_*
+ * IN: Scope, MPI_T_SCOPE_*
+ * IN: get_addr, If not NULL, it is a callback to get address of the cvar.
+ * IN: get_count, If not NULL, it is a callback to read count of the cvar.
+ * IN: cat, Catogery name of the cvar
+ * IN: desc, Description of the cvar
+ */
+void MPIR_T_CVAR_REGISTER_impl(
+    MPI_Datatype dtype, const char* name, const void *addr, int count,
+    MPIR_T_enum_t *etype, MPIR_T_verbosity_t verb, MPIR_T_bind_t bind,
+    MPIR_T_scope_t scope, MPIR_T_cvar_get_addr_cb get_addr,
+    MPIR_T_cvar_get_count_cb get_count, MPIR_T_cvar_value_t defaultval,
+    const char *cat, const char * desc)
+{
+    name2index_hash_t *hash_entry;
+    cvar_table_entry_t *cvar;
+    int cvar_idx;
+
+    /* Check whether this is a replicated cvar, whose name is unique. */
+    HASH_FIND_STR(cvar_hash, name, hash_entry);
+
+    if (hash_entry != NULL) {
+        /* Found it, the cvar already exists */
+        cvar_idx = hash_entry->idx;
+        cvar = (cvar_table_entry_t *)utarray_eltptr(cvar_table, cvar_idx);
+        /* Should never override an existing & active var */
+        MPIU_Assert(cvar->active != TRUE);
+        cvar->active = TRUE;
+        /* FIXME: Do we need to check consistency between the old and new? */
+    } else {
+        /* Not found, so push the cvar to back of cvar_table */
+        utarray_extend_back(cvar_table);
+        cvar = (cvar_table_entry_t *)utarray_back(cvar_table);
+        cvar->active = TRUE;
+        cvar->datatype = dtype;
+        cvar->name = MPIU_Strdup(name);
+        MPIU_Assert(cvar->name);
+        cvar->addr = (void *)addr;
+        cvar->count = count;
+        cvar->verbosity = verb;
+        cvar->bind = bind;
+        cvar->scope = scope;
+        cvar->get_addr = get_addr;
+        cvar->get_count = get_count;
+        cvar->defaultval = defaultval;
+        cvar->desc = MPIU_Strdup(desc);
+        MPIU_Assert(cvar->desc);
+
+        /* Record <name, index> in hash table */
+        cvar_idx = utarray_len(cvar_table) - 1;
+        hash_entry = MPIU_Malloc(sizeof(name2index_hash_t));
+        MPIU_Assert(hash_entry);
+        /* Need not to Strdup name, since cvar_table and cvar_hash co-exist */
+        hash_entry->name =name;
+        hash_entry->idx = cvar_idx;
+        HASH_ADD_KEYPTR(hh, cvar_hash, hash_entry->name,
+                        strlen(hash_entry->name), hash_entry);
+
+        /* Add the cvar to a category */
+        MPIR_T_cat_add_cvar(cat, cvar_idx);
+    }
+}
+
+/* A low level, generic and internally used interface to register
+ * a pvar to MPIR_T. Other modules should use interfaces defined
+ * for concrete pvar classes.
+ *
+ * IN: varclass, MPI_T_PVAR_CLASS_*
+ * IN: dtype, MPI datatype for this pvar
+ * IN: name, Name of the pvar
+ * IN: addr, Pointer to the pvar if known at registeration, otherwise NULL.
+ * IN: count, # of elements of this pvar if known at registeration, otherwise 0.
+ * IN: etype, MPI_T_enum or MPI_T_ENUM_NULL
+ * IN: verb, MPI_T_PVAR_VERBOSITY_*
+ * IN: bind, MPI_T_BIND_*
+ * IN: flags, Bitwise OR of MPIR_T_R_PVAR_FLAGS_{}
+ * IN: get_value, If not NULL, it is a callback to read the pvar.
+ * IN: get_count, If not NULL, it is a callback to read count of the pvar.
+ * IN: cat, Catogery name of the pvar
+ * IN: desc, Description of the pvar
+ */
+void MPIR_T_PVAR_REGISTER_impl(
+    int varclass, MPI_Datatype dtype, const char* name, void *addr, int count,
+    MPIR_T_enum_t *etype, int verb, int bind, int flags,
+    MPIR_T_pvar_get_value_cb get_value, MPIR_T_pvar_get_count_cb get_count,
+    const char * cat, const char * desc)
+{
+    name2index_hash_t *hash_entry;
+    pvar_table_entry_t *pvar;
+    int pvar_idx;
+    int seq = varclass - MPIR_T_PVAR_CLASS_FIRST;
+
+    /* Check whether this is a replicated pvar, whose name is unique per class */
+    HASH_FIND_STR(pvar_hashs[seq], name, hash_entry);
+
+    if (hash_entry != NULL) {
+        /* Found it, the pvar already exists */
+        pvar_idx = hash_entry->idx;
+        pvar = (pvar_table_entry_t *)utarray_eltptr(pvar_table, pvar_idx);
+        /* Should never override an existing & active var */
+        MPIU_Assert(pvar->active != TRUE);
+        pvar->active = TRUE;
+        /* FIXME: Do we need to check consistency between the old and new? */
+    } else {
+        /* Not found, so push the pvar to back of pvar_table */
+        utarray_extend_back(pvar_table);
+        pvar = (pvar_table_entry_t *)utarray_back(pvar_table);
+        pvar->active = TRUE;
+        pvar->varclass = varclass;
+        pvar->datatype = dtype;
+        pvar->name = MPIU_Strdup(name);
+        MPIU_Assert(pvar->name);
+        pvar->addr = addr;
+        pvar->count = count;
+        pvar->enumtype = etype;
+        pvar->verbosity = verb;
+        pvar->bind = bind;
+        pvar->flags = flags;
+        pvar->get_value = get_value;
+        pvar->get_count = get_count;
+        pvar->desc = MPIU_Strdup(desc);
+        MPIU_Assert(pvar->desc);
+
+        /* Record <name, index> in hash table */
+        pvar_idx = utarray_len(pvar_table) - 1;
+        hash_entry = MPIU_Malloc(sizeof(name2index_hash_t));
+        MPIU_Assert(hash_entry);
+        /* Need not to Strdup name, since pvar_table and pvar_hashs co-exist */
+        hash_entry->name = name;
+        hash_entry->idx = pvar_idx;
+        HASH_ADD_KEYPTR(hh, pvar_hashs[seq], hash_entry->name,
+                        strlen(hash_entry->name), hash_entry);
+
+        /* Add the pvar to a category */
+        MPIR_T_cat_add_pvar(cat, utarray_len(pvar_table)-1);
+    }
+}
+
+/* Implements an MPI_T-style strncpy.  Here is the description from the draft
+ * standard:
+ *
+ *   Several MPI tool information interface functions return one or more
+ *   strings. These functions have two arguments for each string to be returned:
+ *   an OUT parameter that identifies a pointer to the buffer in which the
+ *   string will be returned, and an IN/OUT parameter to pass the length of the
+ *   buffer. The user is responsible for the memory allocation of the buffer and
+ *   must pass the size of the buffer (n) as the length argument. Let n be the
+ *   length value specified to the function. On return, the function writes at
+ *   most n - 1 of the string's characters into the buffer, followed by a null
+ *   terminator. If the returned string's length is greater than or equal to n,
+ *   the string will be truncated to n - 1 characters. In this case, the length
+ *   of the string plus one (for the terminating null character) is returned in
+ *   the length argument. If the user passes the null pointer as the buffer
+ *   argument or passes 0 as the length argument, the function does not return
+ *   the string and only returns the length of the string plus one in the length
+ *   argument. If the user passes the null pointer as the length argument, the
+ *   buffer argument is ignored and nothing is returned.
+ *
+ * So this routine copies up to (*len)-1 characters from src to dst and then
+ * sets *len to (strlen(dst)+1).  If dst==NULL, just return (strlen(src)+1) in
+ * *len.
+ *
+ * This routine does not follow MPICH error handling conventions.
+ */
+void MPIR_T_strncpy(char *dst, const char *src, int *len)
+{
+    /* std. says if len arg is NULL, dst is ignored and nothing is returned (MPI-3, p.563) */
+    if (len) {
+        /* If dst is NULL or *len is 0, just return src length + 1 */
+        if (!dst || !*len) {
+            *len = (src == NULL) ? 1 : strlen(src) + 1;
+        }
+        else {
+            /* MPL_strncpy will always terminate the string */
+            MPIU_Assert(*len > 0);
+            if (src != NULL) {
+                MPL_strncpy(dst, src, *len);
+                *len = strlen(dst) + 1;
+            } else {
+                /* As if an empty string is copied */
+                *dst = '\0';
+                *len = 1;
+            }
+        }
+    }
+}
+

http://git.mpich.org/mpich.git/commitdiff/39e5fa1537c97be827a33fe47d9935242b3cf849

commit 39e5fa1537c97be827a33fe47d9935242b3cf849
Author: Junchao Zhang <jczhang at mcs.anl.gov>
Date:   Tue Sep 3 17:09:45 2013 -0500

    Primary implementation of MPI_T specified in MPI3
    
    Implemented the framework for MPI_T enums, categories, performance variables.
    and control variables specified in MPI3. Fix #1898, #1930.
    
    The code should make MPICH MPI_T compliant, though not all MPICH pvars/cvars
    are currently exposed through MPI_T.
    
    The code supports multithreading up to MPI_THREAD_MULTIPLE.
    
    Signed-off-by: Pavan Balaji <balaji at mcs.anl.gov>

diff --git a/src/include/mpi.h.in b/src/include/mpi.h.in
index 6d4bf3b..6ccd9d8 100644
--- a/src/include/mpi.h.in
+++ b/src/include/mpi.h.in
@@ -569,17 +569,18 @@ typedef struct MPI_Status {
 } MPI_Status;
 
 /* types for the MPI_T_ interface */
-struct MPIR_T_enum;
-typedef struct MPIR_T_enum * MPI_T_enum;
-struct MPIR_T_cvar_handle;
-typedef struct MPIR_T_cvar_handle * MPI_T_cvar_handle;
-struct MPIR_T_pvar_handle;
-typedef struct MPIR_T_pvar_handle * MPI_T_pvar_handle;
-struct MPIR_T_pvar_session;
-typedef struct MPIR_T_pvar_session * MPI_T_pvar_session;
+struct MPIR_T_enum_s;
+struct MPIR_T_cvar_handle_s;
+struct MPIR_T_pvar_handle_s;
+struct MPIR_T_pvar_session_s;
+
+typedef struct MPIR_T_enum_s * MPI_T_enum;
+typedef struct MPIR_T_cvar_handle_s * MPI_T_cvar_handle;
+typedef struct MPIR_T_pvar_handle_s * MPI_T_pvar_handle;
+typedef struct MPIR_T_pvar_session_s * MPI_T_pvar_session;
 
 /* extra const at front would be safer, but is incompatible with MPI_T_ prototypes */
-extern struct MPIR_T_pvar_handle * const MPI_T_PVAR_ALL_HANDLES;
+extern struct MPIR_T_pvar_handle_s * const MPI_T_PVAR_ALL_HANDLES;
 
 #define MPI_T_ENUM_NULL         ((MPI_T_enum)NULL)
 #define MPI_T_CVAR_HANDLE_NULL  ((MPI_T_cvar_handle)NULL)
@@ -588,7 +589,7 @@ extern struct MPIR_T_pvar_handle * const MPI_T_PVAR_ALL_HANDLES;
 
 /* the MPI_T_ interface requires that these VERBOSITY constants occur in this
  * relative order with increasing values */
-enum MPIR_T_verbosity_t {
+typedef enum MPIR_T_verbosity_t {
     /* don't name-shift this if/when MPI_T_ is accepted, this is an MPICH-only
      * extension */
     MPIX_T_VERBOSITY_INVALID = 0,
@@ -605,9 +606,9 @@ enum MPIR_T_verbosity_t {
     MPI_T_VERBOSITY_MPIDEV_BASIC,
     MPI_T_VERBOSITY_MPIDEV_DETAIL,
     MPI_T_VERBOSITY_MPIDEV_ALL
-};
+} MPIR_T_verbosity_t;
 
-enum MPIR_T_bind_t {
+typedef enum MPIR_T_bind_t {
     /* don't name-shift this if/when MPI_T_ is accepted, this is an MPICH-only
      * extension */
     MPIX_T_BIND_INVALID = 0,
@@ -624,9 +625,9 @@ enum MPIR_T_bind_t {
     MPI_T_BIND_MPI_WIN,
     MPI_T_BIND_MPI_MESSAGE,
     MPI_T_BIND_MPI_INFO
-};
+} MPIR_T_bind_t;
 
-enum MPIR_T_scope_t {
+typedef enum MPIR_T_scope_t {
     /* don't name-shift this if/when MPI_T_ is accepted, this is an MPICH-only
      * extension */
     MPIX_T_SCOPE_INVALID = 0,
@@ -639,15 +640,16 @@ enum MPIR_T_scope_t {
     MPI_T_SCOPE_GROUP_EQ,
     MPI_T_SCOPE_ALL,
     MPI_T_SCOPE_ALL_EQ
-};
+} MPIR_T_scope_t;
 
-enum MPIR_T_pvar_class_t {
+typedef enum MPIR_T_pvar_class_t {
     /* don't name-shift this if/when MPI_T_ is accepted, this is an MPICH-only
      * extension */
     MPIX_T_PVAR_CLASS_INVALID = 0,
 
     /* arbitrarily shift values to aid debugging and reduce accidental errors */
-    MPI_T_PVAR_CLASS_STATE = 240,
+    MPIR_T_PVAR_CLASS_FIRST = 240,
+    MPI_T_PVAR_CLASS_STATE = MPIR_T_PVAR_CLASS_FIRST,
     MPI_T_PVAR_CLASS_LEVEL,
     MPI_T_PVAR_CLASS_SIZE,
     MPI_T_PVAR_CLASS_PERCENTAGE,
@@ -656,8 +658,10 @@ enum MPIR_T_pvar_class_t {
     MPI_T_PVAR_CLASS_COUNTER,
     MPI_T_PVAR_CLASS_AGGREGATE,
     MPI_T_PVAR_CLASS_TIMER,
-    MPI_T_PVAR_CLASS_GENERIC
-};
+    MPI_T_PVAR_CLASS_GENERIC,
+    MPIR_T_PVAR_CLASS_LAST,
+    MPIR_T_PVAR_CLASS_NUMBER = MPIR_T_PVAR_CLASS_LAST - MPIR_T_PVAR_CLASS_FIRST
+} MPIR_T_pvar_class_t;
 
 /* Handle conversion types/functions */
 
diff --git a/src/include/mpiimpl.h b/src/include/mpiimpl.h
index a14f57c..82f5616 100644
--- a/src/include/mpiimpl.h
+++ b/src/include/mpiimpl.h
@@ -3515,116 +3515,8 @@ typedef struct MPID_Grequest_class {
 } MPID_Grequest_class;
 
 
-/* types and other internal defintions that must be kept out of mpi.h */
-
-/* forward decls */
-struct MPIR_T_pvar_info;
-
-struct MPIR_T_enum {
-    /* TODO replace this struct's contents with a real implementation once we
-     * actually have/support an enum type */
-    int dummy;
-};
-struct MPIR_T_cvar_handle {
-    struct MPIR_Param_t *p;
-};
-struct MPIR_T_pvar_session {
-    /* a utlist-managed list (see mpl_utlist.h) -- _must_ be initialized to NULL
-     * at alloc time or the macros won't work */
-    struct MPIR_T_pvar_handle *hlist;
-
-    /* TODO does anything else need to go in here at this stage? */
-};
-
-
-typedef int MPIR_T_pvar_handle_creator_fn(void *obj_handle,
-                                          struct MPIR_T_pvar_handle *handle,
-                                          int *countp);
-enum MPIR_T_pvar_impl_kind {
-    /* generic read/write impl is fine, just deref pointer */
-    MPIR_T_PVAR_IMPL_SIMPLE,
-
-    /* read/write are dispatched to callbacks to load/store variable values */
-    MPIR_T_PVAR_IMPL_CB
-};
-
-/* These are descriptors that lower level intialization code creates and feeds
- * into the overall MPI_T_pvar_ system in order to permit the upper level code
- * to implement MPI_T_pvar_{get_num,get_info,handle_alloc}. */
-struct MPIR_T_pvar_info {
-    int idx; /* pvar index value for pvar_get_info and friends */
-
-    /* fields for get_info */
-    char *name;
-    enum MPIR_T_verbosity_t verbosity;
-    enum MPIR_T_pvar_class_t varclass;
-    MPI_Datatype dtype;
-    struct MPIR_T_enum *etype;
-    char *desc;
-    enum MPIR_T_bind_t binding;
-    int readonly;
-    int continuous;
-    int atomic;
-
-    /* fields for handle_alloc */
-    enum MPIR_T_pvar_impl_kind impl_kind;
-    void *var_state;
-    MPIR_T_pvar_handle_creator_fn *create_fn;
-};
-
-struct MPIR_T_pvar_handle {
-    /* for linked list of handles attached to the pvar_session */
-    struct MPIR_T_pvar_handle *next;
-    struct MPIR_T_pvar_handle *prev;
-
-    struct MPIR_T_pvar_info *info;
-    struct MPIR_T_pvar_session *session;
-    int count;
-
-    int bytes; /* for _IMPL_SIMPLE */
-
-    /* for _IMPL_CB types this vtable prevents us from having to duplicate
-     * multiple sets of function pointers in each handle at the expense of an
-     * extra pointer indirection */
-    struct MPIR_T_pvar_hnd_vtable *vtable;
-    void *handle_state;
-    int free_handle_state; /* boolean -- true iff the "handle_state" pointer
-                            * should be freed when this handle is freed */
-};
-
-/* vtable structure for handle "objects".  Implements all major handle operations */
-struct MPIR_T_pvar_hnd_vtable {
-    int (*free)(struct MPIR_T_pvar_session *session, struct MPIR_T_pvar_handle *handle);
-    int (*start)(struct MPIR_T_pvar_session *session, struct MPIR_T_pvar_handle *handle);
-    int (*stop)(struct MPIR_T_pvar_session *session, struct MPIR_T_pvar_handle *handle);
-    int (*read)(struct MPIR_T_pvar_session *session, struct MPIR_T_pvar_handle *handle, void *buf);
-    int (*write)(struct MPIR_T_pvar_session *session, struct MPIR_T_pvar_handle *handle, void *buf);
-    int (*reset)(struct MPIR_T_pvar_session *session, struct MPIR_T_pvar_handle *handle);
-    int (*readreset)(struct MPIR_T_pvar_session *session, struct MPIR_T_pvar_handle *handle, void *buf);
-};
-
-/* Called by lower-level initialization code to add pvars to the global list.
- * Will cause the value returned by MPI_T_pvar_get_num to be incremented and
- * sets up that new index to work with get_info, handle_alloc, etc. */
-int MPIR_T_pvar_add(const char *name,
-                    enum MPIR_T_verbosity_t verbosity,
-                    enum MPIR_T_pvar_class_t varclass,
-                    MPI_Datatype dtype,
-                    struct MPIR_T_enum *enumtype,
-                    const char *desc,
-                    enum MPIR_T_bind_t bind,
-                    int readonly,
-                    int continuous,
-                    int atomic,
-                    enum MPIR_T_pvar_impl_kind impl_kind,
-                    void *var_state,
-                    MPIR_T_pvar_handle_creator_fn *create_fn,
-                    int *index_p);
-
-int MPIR_T_get_num_pvars(int *num);
-int MPIR_T_get_pvar_info_by_idx(int idx, struct MPIR_T_pvar_info **info_p);
-int MPIR_T_finalize_pvars(void);
-void MPIU_Tool_strncpy(char *dst, const char *src, int *len);
+/* Interfaces exposed by MPI_T */
+#include "mpit.h"
 
 /*TTopoOverview.tex
  *
@@ -4288,18 +4180,9 @@ int MPIR_Ialltoallw_intra(const void *sendbuf, const int *sendcounts, const int
 int MPIR_Ialltoallw_inter(const void *sendbuf, const int *sendcounts, const int *sdispls, const MPI_Datatype *sendtypes, void *recvbuf, const int *recvcounts, const int *rdispls, const MPI_Datatype *recvtypes, MPID_Comm *comm_ptr, MPID_Sched_t s);
 
 /* begin impl functions for MPI_T (MPI_T_ right now) */
-int MPIR_T_init_thread_impl(int required, int *provided);
-int MPIR_T_finalize_impl(void);
-int MPIR_T_enum_get_info_impl(MPI_T_enum enumtype, int *num, char *name, int *name_len);
-int MPIR_T_enum_get_item_impl(MPI_T_enum enumtype, int num, int *value, char *name, int *name_len);
-int MPIR_T_cvar_get_num_impl(int *num_cvar);
-int MPIR_T_cvar_get_info_impl(int cvar_index, char *name, int *name_len, int *verbosity, MPI_Datatype *datatype, MPI_T_enum *enumtype, char *desc, int *desc_len, int *binding, int *scope);
 int MPIR_T_cvar_handle_alloc_impl(int cvar_index, void *obj_handle, MPI_T_cvar_handle *handle, int *count);
-int MPIR_T_cvar_handle_free_impl(MPI_T_cvar_handle *handle);
 int MPIR_T_cvar_read_impl(MPI_T_cvar_handle handle, void *buf);
 int MPIR_T_cvar_write_impl(MPI_T_cvar_handle handle, void *buf);
-int MPIR_T_pvar_get_num_impl(int *num_pvar);
-int MPIR_T_pvar_get_info_impl(int pvar_index, char *name, int *name_len, int *verbosity, int *var_class, MPI_Datatype *datatype, MPI_T_enum *enumtype, char *desc, int *desc_len, int *binding, int *readonly, int *continuous, int *atomic);
 int MPIR_T_pvar_session_create_impl(MPI_T_pvar_session *session);
 int MPIR_T_pvar_session_free_impl(MPI_T_pvar_session *session);
 int MPIR_T_pvar_handle_alloc_impl(MPI_T_pvar_session session, int pvar_index, void *obj_handle, MPI_T_pvar_handle *handle, int *count);
@@ -4310,12 +4193,9 @@ int MPIR_T_pvar_read_impl(MPI_T_pvar_session session, MPI_T_pvar_handle handle,
 int MPIR_T_pvar_write_impl(MPI_T_pvar_session session, MPI_T_pvar_handle handle, void *buf);
 int MPIR_T_pvar_reset_impl(MPI_T_pvar_session session, MPI_T_pvar_handle handle);
 int MPIR_T_pvar_readreset_impl(MPI_T_pvar_session session, MPI_T_pvar_handle handle, void *buf);
-int MPIR_T_category_get_num_impl(int *num_cat);
-int MPIR_T_category_get_info_impl(int cat_index, char *name, int *name_len, char *desc, int *desc_len, int *num_controlvars, int *num_pvars, int *num_categories);
 int MPIR_T_category_get_cvars_impl(int cat_index, int len, int indices[]);
 int MPIR_T_category_get_pvars_impl(int cat_index, int len, int indices[]);
 int MPIR_T_category_get_categories_impl(int cat_index, int len, int indices[]);
-int MPIR_T_category_changed_impl(int *stamp);
 /* end impl functions for MPI_T (MPI_T_ right now) */
 
 /* MPI-3 "large count" impl routines */
@@ -4325,8 +4205,6 @@ void MPIR_Type_get_extent_x_impl(MPI_Datatype datatype, MPI_Count *lb, MPI_Count
 void MPIR_Type_get_true_extent_x_impl(MPI_Datatype datatype, MPI_Count *true_lb, MPI_Count *true_extent);
 int MPIR_Type_size_x_impl(MPI_Datatype datatype, MPI_Count *size);
 
-int MPIR_T_is_initialized(void);
-
 /* random initializers */
 int MPIR_Group_init(void);
 int MPIR_Comm_init(MPID_Comm *);
diff --git a/src/include/mpit.h b/src/include/mpit.h
new file mode 100644
index 0000000..ea97e31
--- /dev/null
+++ b/src/include/mpit.h
@@ -0,0 +1,623 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
+/*
+ *  (C) 2013 by Argonne National Laboratory.
+ *      See COPYRIGHT in top-level directory.
+ */
+
+/* Interfaces in this file are intended to be used by modules other than
+ * MPIR_T. They use these interfaces to declare, create, operate and expose
+ * enums, control variables, performance variables and categories.
+ */
+#ifndef MPIT_H_INCLUDED
+#define MPIT_H_INCLUDED
+
+#include "mpitimpl.h"
+
+/* Stamp to indicate a change in categories */
+extern int cat_stamp;
+
+/* Tables to store enums, categories, cvars, pvars.
+ * All tables need random access except enum_table. But we still implement
+ * enum_table as an array, since it is handy and element size of
+ * enum_table (i.e., sizeof(enum_item_t)) is small.
+ */
+extern UT_array *enum_table;
+extern UT_array *cat_table;
+extern UT_array *cvar_table;
+extern UT_array *pvar_table;
+
+/* Hash tables to quick locate category, cvar, pvar indices by their names */
+extern name2index_hash_t *cat_hash;
+extern name2index_hash_t *cvar_hash;
+/* pvar names are unique per pvar class. So we use multiple hashs */
+extern name2index_hash_t *pvar_hashs[MPIR_T_PVAR_CLASS_NUMBER];
+
+/* See description in mpit.c */
+extern void MPIR_T_enum_create(const char *name, MPI_T_enum *handle);
+extern void MPIR_T_enum_add_item(MPI_T_enum handle, const char *item_name, int item_value);
+extern int MPIR_T_cat_add_pvar(const char *cat_name, int pvar_index);
+extern int MPIR_T_cat_add_cvar(const char *cat_name, int cvar_index);
+extern int MPIR_T_cat_add_subcat(const char *parent_name, const char *child_name);
+extern int MPIR_T_cat_add_desc(const char *cat_name, const char *cat_desc);
+
+static inline cvar_table_entry_t * LOOKUP_CVAR_BY_NAME(const char* cvar_name)
+{
+    name2index_hash_t *hash_entry;
+    HASH_FIND_STR(cvar_hash, cvar_name, hash_entry);
+    MPIU_Assert(hash_entry != NULL);
+    int cvar_idx = hash_entry->idx;
+    return (cvar_table_entry_t *)utarray_eltptr(cvar_table, cvar_idx);
+}
+
+/* Helper macros for getting the default value of a cvar */
+#define MPIR_PARAM_GET_DEFAULT_INT(name_,out_ptr_)   \
+    do {  \
+        cvar_table_entry_t *cvar = LOOKUP_CVAR_BY_NAME(#name_); \
+        *(out_ptr_) = cvar->defaultval.d; \
+    } while (0)
+
+#define MPIR_PARAM_GET_DEFAULT_BOOLEAN(name_,out_ptr_)   \
+    do {  \
+        MPIR_PARAM_GET_DEFAULT_INT(name_,out_ptr_); \
+    } while (0)
+
+#define MPIR_PARAM_GET_DEFAULT_UNSIGNED(name_,out_ptr_)   \
+    do {  \
+        cvar_table_entry_t *cvar = LOOKUP_CVAR_BY_NAME(#name_); \
+        *(out_ptr_) = cvar->defaultval.u; \
+    } while (0)
+
+#define MPIR_PARAM_GET_DEFAULT_UNSIGNED_LONG(name_,out_ptr_)   \
+    do {  \
+        cvar_table_entry_t *cvar = LOOKUP_CVAR_BY_NAME(#name_); \
+        *(out_ptr_) = cvar->defaultval.ul; \
+    } while (0)
+
+#define MPIR_PARAM_GET_DEFAULT_UNSIGNED_LONG_LONG(name_,out_ptr_)   \
+    do {  \
+        cvar_table_entry_t *cvar = LOOKUP_CVAR_BY_NAME(#name_); \
+        *(out_ptr_) = cvar->defaultval.ull; \
+    } while (0)
+
+#define MPIR_PARAM_GET_DEFAULT_DOUBLE(name_,out_ptr_)   \
+    do {  \
+        cvar_table_entry_t *cvar = LOOKUP_CVAR_BY_NAME(#name_); \
+        *(out_ptr_) = cvar->defaultval.f; \
+    } while (0)
+
+#define MPIR_PARAM_GET_DEFAULT_STRING(name_,out_ptr_)   \
+    do {  \
+        cvar_table_entry_t *cvar = LOOKUP_CVAR_BY_NAME(#name_); \
+        *(out_ptr_) = cvar->defaultval.str; \
+    } while (0)
+
+#define MPIR_PARAM_GET_DEFAULT_RANGE(name_,out_ptr_)   \
+    do {  \
+        cvar_table_entry_t *cvar = LOOKUP_CVAR_BY_NAME(#name_); \
+        *(out_ptr_) = cvar->defaultval.range; \
+    } while (0)
+
+/* Register a static cvar. A static pvar has NO binding and its address
+ * and count are known at registeration time.
+ * Attention: name_ is a token not a string
+*/
+#define MPIR_T_CVAR_REGISTER_STATIC(dtype_, name_, addr_, count_, verb_, \
+            scope_, default_, cat_, desc_) \
+    do { \
+        MPIU_Assert(addr_ != NULL); \
+        MPIU_Assert(count_ > 0); \
+        MPIR_T_CVAR_REGISTER_impl(dtype_, #name_, addr_, count_, MPI_T_ENUM_NULL, \
+            verb_, MPI_T_BIND_NO_OBJECT, scope_, NULL, NULL, default_, cat_, desc_); \
+    } while (0)
+
+/* Register a dynamic cvar, which may have object binding and whose
+ * address and count may be unknown at registeration time. It is
+ * developers' duty to provide self-contained arguments.
+*/
+#define MPIR_T_CVAR_REGISTER_DYNAMIC(dtype_, name_, addr_, count_, etype_, \
+            verb_, bind_, scope_, get_addr_, get_count_, default_, cat_, desc_) \
+    do { \
+        MPIU_Assert(addr_ != NULL || get_addr_ != NULL); \
+        MPIU_Assert(count_ > 0 || get_count_ != NULL); \
+        MPIR_T_CVAR_REGISTER_impl(dtype_, #name_, addr_, count_, etype_, \
+            verb_, bind_, scope_, get_addr_, get_count_, default_, cat_, desc_); \
+    } while (0)
+
+
+/* The following are interfaces for each pvar classe,
+ * basically including delcaration, access and registeration.
+ */
+
+/* STATE */
+#define MPIR_T_PVAR_INT_STATE_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_INT_STATE_DECL_impl(name_))
+
+#define MPIR_T_PVAR_INT_STATE_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_INT_STATE_DECL_STATIC_impl(name_))
+
+#define MPIR_T_PVAR_INT_STATE_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_INT_STATE_DECL_EXTERN_impl(name_))
+
+#define MPIR_T_PVAR_STATE_SET_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_STATE_SET_VAR_impl(ptr_, val_))
+#define MPIR_T_PVAR_STATE_GET_VAR(MODULE, ptr_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_STATE_GET_VAR_impl(ptr_))
+
+#define MPIR_T_PVAR_STATE_SET(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_STATE_SET_impl(name_, val_))
+#define MPIR_T_PVAR_STATE_GET(MODULE, name_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_STATE_GET_impl(name_))
+
+#define MPIR_T_PVAR_STATE_REGISTER_STATIC(MODULE, dtype_, name_, \
+            initval_, etype_, verb_, bind_, flags_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_STATE_REGISTER_STATIC_impl(dtype_, name_, \
+            initval_, etype_, verb_, bind_, flags_, cat_, desc_))
+
+#define MPIR_T_PVAR_STATE_REGISTER_DYNAMIC(MODULE, dtype_, name_, \
+            addr_, count_, etype_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_STATE_REGISTER_DYNAMIC_impl(dtype_, name_, \
+            addr_, count_, etype_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_))
+
+/* LEVEL */
+#define MPIR_T_PVAR_UINT_LEVEL_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_LEVEL_DECL_impl(name_))
+#define MPIR_T_PVAR_ULONG_LEVEL_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_LEVEL_DECL_impl(name_))
+#define MPIR_T_PVAR_ULONG2_LEVEL_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_LEVEL_DECL_impl(name_))
+#define MPIR_T_PVAR_DOUBLE_LEVEL_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_LEVEL_DECL_impl(name_))
+
+#define MPIR_T_PVAR_UINT_LEVEL_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_LEVEL_DECL_STATIC_impl(name_))
+#define MPIR_T_PVAR_ULONG_LEVEL_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_LEVEL_DECL_STATIC_impl(name_))
+#define MPIR_T_PVAR_ULONG2_LEVEL_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_LEVEL_DECL_STATIC_impl(name_))
+#define MPIR_T_PVAR_DOUBLE_LEVEL_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_LEVEL_DECL_STATIC_impl(name_))
+
+#define MPIR_T_PVAR_UINT_LEVEL_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_LEVEL_DECL_EXTERN_impl(name_))
+#define MPIR_T_PVAR_ULONG_LEVEL_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_LEVEL_DECL_EXTERN_impl(name_))
+#define MPIR_T_PVAR_ULONG2_LEVEL_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_LEVEL_DECL_EXTERN_impl(name_))
+#define MPIR_T_PVAR_DOUBLE_LEVEL_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_LEVEL_DECL_EXTERN_impl(name_))
+
+#define MPIR_T_PVAR_LEVEL_SET_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_LEVEL_SET_VAR_impl(ptr_, val_))
+#define MPIR_T_PVAR_LEVEL_GET_VAR(MODULE, ptr_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_LEVEL_GET_VAR_impl(ptr_))
+#define MPIR_T_PVAR_LEVEL_INC_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_LEVEL_INC_VAR_impl(ptr_, val_))
+#define MPIR_T_PVAR_LEVEL_DEC_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_LEVEL_DEC_VAR_impl(ptr_, val_))
+
+#define MPIR_T_PVAR_LEVEL_SET(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_LEVEL_SET_impl(name_, val_))
+#define MPIR_T_PVAR_LEVEL_GET(MODULE, name_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_LEVEL_GET_impl(name_))
+#define MPIR_T_PVAR_LEVEL_INC(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_LEVEL_INC_impl(name_, val_))
+#define MPIR_T_PVAR_LEVEL_DEC(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_LEVEL_DEC_impl(name_, val_))
+
+#define MPIR_T_PVAR_LEVEL_REGISTER_STATIC(MODULE, dtype_, name_, \
+            initval_, verb_, bind_, flags_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_LEVEL_REGISTER_STATIC_impl(dtype_, name_, \
+            initval_, verb_, bind_, flags_, cat_, desc_))
+
+#define MPIR_T_PVAR_LEVEL_REGISTER_DYNAMIC(MODULE, dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_LEVEL_REGISTER_DYNAMIC_impl(dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count, cat_, desc_))
+
+/* SIZE */
+#define MPIR_T_PVAR_UINT_SIZE_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_SIZE_DECL_impl(name_))
+#define MPIR_T_PVAR_ULONG_SIZE_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_SIZE_DECL_impl(name_))
+#define MPIR_T_PVAR_ULONG2_SIZE_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_SIZE_DECL_impl(name_))
+#define MPIR_T_PVAR_DOUBLE_SIZE_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_SIZE_DECL_impl(name_))
+
+#define MPIR_T_PVAR_UINT_SIZE_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_SIZE_DECL_STATIC_impl(name_))
+#define MPIR_T_PVAR_ULONG_SIZE_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_SIZE_DECL_STATIC_impl(name_))
+#define MPIR_T_PVAR_ULONG2_SIZE_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_SIZE_DECL_STATIC_impl(name_))
+#define MPIR_T_PVAR_DOUBLE_SIZE_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_SIZE_DECL_STATIC_impl(name_))
+
+#define MPIR_T_PVAR_UINT_SIZE_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_SIZE_DECL_EXTERN_impl(name_))
+#define MPIR_T_PVAR_ULONG_SIZE_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_SIZE_DECL_EXTERN_impl(name_))
+#define MPIR_T_PVAR_ULONG2_SIZE_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_SIZE_DECL_EXTERN_impl(name_))
+#define MPIR_T_PVAR_DOUBLE_SIZE_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_SIZE_DECL_EXTERN_impl(name_))
+
+#define MPIR_T_PVAR_SIZE_SET_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_SIZE_SET_VAR_impl(ptr_, val_))
+#define MPIR_T_PVAR_SIZE_GET_VAR(MODULE, ptr_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_SIZE_GET_VAR_impl(ptr_))
+
+#define MPIR_T_PVAR_SIZE_SET(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_SIZE_SET_impl(name_, val_))
+#define MPIR_T_PVAR_SIZE_GET(MODULE, name_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_SIZE_GET_impl(name_))
+
+#define MPIR_T_PVAR_SIZE_REGISTER_STATIC(MODULE, dtype_, name_, \
+            initval_, verb_, bind_, flags_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_SIZE_REGISTER_STATIC_impl(dtype_, name_, \
+            initval_, verb_, bind_, flags_, cat_, desc_))
+
+#define MPIR_T_PVAR_SIZE_REGISTER_DYNAMIC(MODULE, dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_SIZE_REGISTER_DYNAMIC_impl(dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count, cat_, desc_))
+
+/* PERCENTAGE */
+#define MPIR_T_PVAR_DOUBLE_PERCENTAGE_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_PERCENTAGE_DECL_impl(name_))
+
+#define MPIR_T_PVAR_DOUBLE_PERCENTAGE_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_PERCENTAGE_DECL_STATIC_impl(name_))
+
+#define MPIR_T_PVAR_DOUBLE_PERCENTAGE_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_PERCENTAGE_DECL_EXTERN_impl(name_))
+
+#define MPIR_T_PVAR_PERCENTAGE_SET_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_PERCENTAGE_SET_VAR_impl(ptr_, val_))
+#define MPIR_T_PVAR_PERCENTAGE_GET_VAR(MODULE, ptr_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_PERCENTAGE_GET_VAR_impl(ptr_))
+
+#define MPIR_T_PVAR_PERCENTAGE_SET(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_PERCENTAGE_SET_impl(name_, val_))
+#define MPIR_T_PVAR_PERCENTAGE_GET(MODULE, name_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_PERCENTAGE_GET_impl(name_))
+
+#define MPIR_T_PVAR_PERCENTAGE_REGISTER_STATIC(MODULE, dtype_, name_, \
+            initval_, verb_, bind_, flags_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_PERCENTAGE_REGISTER_STATIC_impl(dtype_, name_, \
+            initval_, verb_, bind_, flags_, cat_, desc_))
+
+#define MPIR_T_PVAR_PERCENTAGE_REGISTER_DYNAMIC(MODULE, dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_PERCENTAGE_REGISTER_DYNAMIC_impl(dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count, cat_, desc_))
+
+/* COUNTER */
+#define MPIR_T_PVAR_UINT_COUNTER_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_COUNTER_DECL_impl(name_))
+#define MPIR_T_PVAR_ULONG_COUNTER_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_COUNTER_DECL_impl(name_))
+#define MPIR_T_PVAR_ULONG2_COUNTER_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_COUNTER_DECL_impl(name_))
+
+#define MPIR_T_PVAR_UINT_COUNTER_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_COUNTER_DECL_STATIC_impl(name_))
+#define MPIR_T_PVAR_ULONG_COUNTER_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_COUNTER_DECL_STATIC_impl(name_))
+#define MPIR_T_PVAR_ULONG2_COUNTER_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_COUNTER_DECL_STATIC_impl(name_))
+
+#define MPIR_T_PVAR_UINT_COUNTER_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_COUNTER_DECL_EXTERN_impl(name_))
+#define MPIR_T_PVAR_ULONG_COUNTER_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_COUNTER_DECL_EXTERN_impl(name_))
+#define MPIR_T_PVAR_ULONG2_COUNTER_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_COUNTER_DECL_EXTERN_impl(name_))
+
+#define MPIR_T_PVAR_COUNTER_INIT_VAR(MODULE, ptr_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_COUNTER_INIT_VAR_impl(ptr_))
+#define MPIR_T_PVAR_COUNTER_GET_VAR(MODULE, ptr_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_COUNTER_GET_VAR_impl(ptr_))
+#define MPIR_T_PVAR_COUNTER_INC_VAR(MODULE, ptr_, inc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_COUNTER_INC_VAR_impl(ptr_, inc_))
+
+#define MPIR_T_PVAR_COUNTER_INIT(MODULE, name_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_COUNTER_INIT_impl(name_))
+#define MPIR_T_PVAR_COUNTER_GET(MODULE, name_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_COUNTER_GET_impl(name_))
+#define MPIR_T_PVAR_COUNTER_INC(MODULE, name_, inc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_COUNTER_INC_impl(name_, inc_))
+
+#define MPIR_T_PVAR_COUNTER_REGISTER_STATIC(MODULE, dtype_, name_, \
+            verb_, bind_, flags_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_COUNTER_REGISTER_STATIC_impl(dtype_, name_, \
+            verb_, bind_, flags_, cat_, desc_))
+
+#define MPIR_T_PVAR_COUNTER_REGISTER_DYNAMIC(MODULE, dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_COUNTER_REGISTER_DYNAMIC_impl(dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_))
+
+/* COUNTER ARRAY for user's convenience */
+#define MPIR_T_PVAR_UINT_COUNTER_ARRAY_DECL(MODULE, name_, len_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_UINT_COUNTER_ARRAY_DECL_impl(name_, len_))
+#define MPIR_T_PVAR_ULONG_COUNTER_ARRAY_DECL(MODULE, name_, len_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG_COUNTER_ARRAY_DECL_impl(name_, len_))
+#define MPIR_T_PVAR_ULONG2_COUNTER_ARRAY_DECL(MODULE, name_, len_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG2_COUNTER_ARRAY_DECL_impl(name_, len_))
+
+#define MPIR_T_PVAR_UINT_COUNTER_ARRAY_DECL_STATIC(MODULE, name_, len_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_UINT_COUNTER_ARRAY_DECL_STATIC_impl(name_, len_))
+#define MPIR_T_PVAR_ULONG_COUNTER_ARRAY_DECL_STATIC(MODULE, name_, len_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG_COUNTER_ARRAY_DECL_STATIC_impl(name_, len_))
+#define MPIR_T_PVAR_ULONG2_COUNTER_ARRAY_DECL_STATIC(MODULE, name_, len_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG2_COUNTER_ARRAY_DECL_STATIC_impl(name_, len_))
+
+#define MPIR_T_PVAR_UINT_COUNTER_ARRAY_DECL_EXTERN(MODULE, name_, len_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_UINT_COUNTER_ARRAY_DECL_EXTERN_impl(name_, len_))
+#define MPIR_T_PVAR_ULONG_COUNTER_ARRAY_DECL_EXTERN(MODULE, name_, len_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG_COUNTER_ARRAY_DECL_EXTERN_impl(name_, len_))
+#define MPIR_T_PVAR_ULONG2_COUNTER_ARRAY_DECL_EXTERN(MODULE, name_, len_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG2_COUNTER_ARRAY_DECL_EXTERN_impl(name_, len_))
+
+#define MPIR_T_PVAR_COUNTER_ARRAY_INIT_VAR(MODULE, ptr_, count_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_COUNTER_ARRAY_INIT_VAR_impl(ptr_, count_))
+#define MPIR_T_PVAR_COUNTER_ARRAY_GET_VAR(MODULE, ptr_, idx_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_COUNTER_ARRAY_GET_VAR_impl(ptr_, idx_))
+#define MPIR_T_PVAR_COUNTER_ARRAY_INC_VAR(MODULE, ptr_, idx_, inc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_COUNTER_ARRAY_INC_VAR_impl(ptr_, idx_, inc_))
+
+#define MPIR_T_PVAR_COUNTER_ARRAY_INIT(MODULE, name_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_COUNTER_ARRAY_INIT_impl(name_))
+#define MPIR_T_PVAR_COUNTER_ARRAY_GET(MODULE, name_, idx_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_COUNTER_ARRAY_GET_impl(name_, idx_))
+#define MPIR_T_PVAR_COUNTER_ARRAY_INC(MODULE, ptr_, idx_, inc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_COUNTER_ARRAY_INC_impl(ptr_, idx_, inc_))
+
+#define MPIR_T_PVAR_COUNTER_ARRAY_REGISTER_STATIC(MODULE, dtype_, name_, \
+            verb_, bind_, flags_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_COUNTER_ARRAY_REGISTER_STATIC_impl(dtype_, name_, \
+            verb_, bind_, flags_, cat_, desc_))
+
+/* ARRGEGATE */
+#define MPIR_T_PVAR_UINT_AGGREGATE_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_AGGREGATE_DECL_impl(name_))
+#define MPIR_T_PVAR_ULONG_AGGREGATE_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_AGGREGATE_DECL_impl(name_))
+#define MPIR_T_PVAR_ULONG2_AGGREGATE_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_AGGREGATE_DECL_impl(name_))
+#define MPIR_T_PVAR_ULONG2_AGGREGATE_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_AGGREGATE_DECL_impl(name_))
+
+#define MPIR_T_PVAR_UINT_AGGREGATE_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_AGGREGATE_DECL_STATIC_impl(name_))
+#define MPIR_T_PVAR_ULONG_AGGREGATE_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_AGGREGATE_DECL_STATIC_impl(name_))
+#define MPIR_T_PVAR_ULONG2_AGGREGATE_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_AGGREGATE_DECL_STATIC_impl(name_))
+#define MPIR_T_PVAR_ULONG2_AGGREGATE_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_AGGREGATE_DECL_STATIC_impl(name_))
+
+#define MPIR_T_PVAR_UINT_AGGREGATE_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_AGGREGATE_DECL_EXTERN_impl(name_))
+#define MPIR_T_PVAR_ULONG_AGGREGATE_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_AGGREGATE_DECL_EXTERN_impl(name_))
+#define MPIR_T_PVAR_ULONG2_AGGREGATE_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_AGGREGATE_DECL_EXTERN_impl(name_))
+#define MPIR_T_PVAR_ULONG2_AGGREGATE_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_AGGREGATE_DECL_EXTERN_impl(name_))
+
+#define MPIR_T_PVAR_AGGREGATE_INIT_VAR(MODULE, ptr_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_AGGREGATE_INIT_VAR_impl(ptr_))
+#define MPIR_T_PVAR_AGGREGATE_GET_VAR(MODULE, ptr_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_AGGREGATE_GET_VAR_impl(ptr_))
+#define MPIR_T_PVAR_AGGREGATE_INC_VAR(MODULE, ptr_, inc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_AGGREGATE_INC_VAR_impl(ptr_, inc_))
+
+#define MPIR_T_PVAR_AGGREGATE_INIT(MODULE, name_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_AGGREGATE_INIT_impl(name_))
+#define MPIR_T_PVAR_AGGREGATE_GET(MODULE, name_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_AGGREGATE_GET_impl(name_))
+#define MPIR_T_PVAR_AGGREGATE_INC(MODULE, name_, inc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_AGGREGATE_INC_impl(name_, inc_))
+
+#define MPIR_T_PVAR_AGGREGATE_REGISTER_STATIC(MODULE, dtype_, name_, \
+            verb_, bind_, flags_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_AGGREGATE_REGISTER_STATIC_impl(dtype_, name_, \
+            verb_, bind_, flags_, cat_, desc_))
+
+#define MPIR_T_PVAR_AGGREGATE_REGISTER_DYNAMIC(MODULE, dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_AGGREGATE_REGISTER_DYNAMIC_impl(dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_))
+
+/* TIMER */
+#define MPIR_T_PVAR_DOUBLE_TIMER_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_TIMER_DECL_impl(name_))
+
+#define MPIR_T_PVAR_DOUBLE_TIMER_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_TIMER_DECL_STATIC_impl(name_))
+
+#define MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN_impl(name_))
+
+#define MPIR_T_PVAR_TIMER_INIT_VAR(MODULE, ptr_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_TIMER_INIT_VAR_impl(ptr_))
+#define MPIR_T_PVAR_TIMER_GET_VAR(MODULE, ptr_, buf) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_TIMER_GET_VAR_impl(ptr_, buf))
+#define MPIR_T_PVAR_TIMER_START_VAR(MODULE, ptr_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_TIMER_START_VAR_impl(ptr_))
+#define MPIR_T_PVAR_TIMER_END_VAR(MODULE, ptr_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_TIMER_END_VAR_impl(ptr_))
+
+#define MPIR_T_PVAR_TIMER_INIT(MODULE, name_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_TIMER_INIT_impl(name_))
+#define MPIR_T_PVAR_TIMER_GET(MODULE, name_, buf_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_TIMER_GET_impl(name_, buf_))
+#define MPIR_T_PVAR_TIMER_START(MODULE, name_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_TIMER_START_impl(name_))
+#define MPIR_T_PVAR_TIMER_END(MODULE, name_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_TIMER_END_impl(name_))
+
+#define MPIR_T_PVAR_TIMER_REGISTER_STATIC(MODULE, dtype_, name_, \
+            verb_, bind_, flags_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_TIMER_REGISTER_STATIC_impl(dtype_, name_, \
+            verb_, bind_, flags_, cat_, desc_))
+
+#define MPIR_T_PVAR_TIMER_REGISTER_DYNAMIC(MODULE, dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_TIMER_REGISTER_DYNAMIC_impl(dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_))
+
+/* HIGHWATERMARK */
+#define MPIR_T_PVAR_UINT_HIGHWATERMARK_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_HIGHWATERMARK_DECL_impl(name_))
+#define MPIR_T_PVAR_ULONG_HIGHWATERMARK_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_HIGHWATERMARK_DECL_impl(name_))
+#define MPIR_T_PVAR_ULONG2_HIGHWATERMARK_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_HIGHWATERMARK_DECL_impl(name_))
+#define MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_DECL_impl(name_))
+
+#define MPIR_T_PVAR_UINT_HIGHWATERMARK_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_HIGHWATERMARK_DECL_STATIC_impl(name_))
+#define MPIR_T_PVAR_ULONG_HIGHWATERMARK_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_HIGHWATERMARK_DECL_STATIC_impl(name_))
+#define MPIR_T_PVAR_ULONG2_HIGHWATERMARK_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_HIGHWATERMARK_DECL_STATIC_impl(name_))
+#define MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_DECL_STATIC_impl(name_))
+
+#define MPIR_T_PVAR_UINT_HIGHWATERMARK_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_HIGHWATERMARK_DECL_EXTERN_impl(name_))
+#define MPIR_T_PVAR_ULONG_HIGHWATERMARK_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_HIGHWATERMARK_DECL_EXTERN_impl(name_))
+#define MPIR_T_PVAR_ULONG2_HIGHWATERMARK_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_HIGHWATERMARK_DECL_EXTERN_impl(name_))
+#define MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_DECL_EXTERN_impl(name_))
+
+/* Embed type names in watermark operations because these operations need to
+   know types of watermark to work, however we can't use typeof(ptr_) in C.
+ */
+#define MPIR_T_PVAR_UINT_HIGHWATERMARK_INIT_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_UINT_HIGHWATERMARK_INIT_VAR_impl(ptr_, val_))
+#define MPIR_T_PVAR_ULONG_HIGHWATERMARK_INIT_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG_HIGHWATERMARK_INIT_VAR_impl(ptr_, val_))
+#define MPIR_T_PVAR_ULONG2_HIGHWATERMARK_INIT_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG2_HIGHWATERMARK_INIT_VAR_impl(ptr_, val_))
+#define MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_INIT_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_INIT_VAR_impl(ptr_, val_))
+
+#define MPIR_T_PVAR_UINT_HIGHWATERMARK_UPDATE_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_UINT_HIGHWATERMARK_UPDATE_VAR_impl(ptr_, val_))
+#define MPIR_T_PVAR_ULONG_HIGHWATERMARK_UPDATE_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG_HIGHWATERMARK_UPDATE_VAR_impl(ptr_, val_))
+#define MPIR_T_PVAR_ULONG2_HIGHWATERMARK_UPDATE_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG2_HIGHWATERMARK_UPDATE_VAR_impl(ptr_, val_))
+#define MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_UPDATE_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_UPDATE_VAR_impl(ptr_, val_))
+
+#define MPIR_T_PVAR_UINT_HIGHWATERMARK_INIT(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_UINT_HIGHWATERMARK_INIT_impl(name_, val_))
+#define MPIR_T_PVAR_ULONG_HIGHWATERMARK_INIT(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG_HIGHWATERMARK_INIT_impl(name_, val_))
+#define MPIR_T_PVAR_ULONG2_HIGHWATERMARK_INIT(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG2_HIGHWATERMARK_INIT_impl(name_, val_))
+#define MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_INIT(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_INIT_impl(name_, val_))
+
+#define MPIR_T_PVAR_UINT_HIGHWATERMARK_UPDATE(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_UINT_HIGHWATERMARK_UPDATE_impl(name_, val_))
+#define MPIR_T_PVAR_ULONG_HIGHWATERMARK_UPDATE(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG_HIGHWATERMARK_UPDATE_impl(name_, val_))
+#define MPIR_T_PVAR_ULONG2_HIGHWATERMARK_UPDATE(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG2_HIGHWATERMARK_UPDATE_impl(name_, val_))
+#define MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_UPDATE(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_UPDATE_impl(name_, val_))
+
+#define MPIR_T_PVAR_HIGHWATERMARK_REGISTER_STATIC(MODULE, dtype_, \
+            name_, initval_, verb_, bind_, flags_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_HIGHWATERMARK_REGISTER_STATIC_impl(dtype_, name_, \
+            initval_, verb_, bind_, flags_, cat_, desc_))
+
+#define MPIR_T_PVAR_HIGHWATERMARK_REGISTER_DYNAMIC(MODULE, dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_HIGHWATERMARK_REGISTER_DYNAMIC_impl(dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_))
+
+/* LOWWATERMARK */
+#define MPIR_T_PVAR_UINT_LOWWATERMARK_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_LOWWATERMARK_DECL_impl(name_))
+#define MPIR_T_PVAR_ULONG_LOWWATERMARK_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_LOWWATERMARK_DECL_impl(name_))
+#define MPIR_T_PVAR_ULONG2_LOWWATERMARK_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_LOWWATERMARK_DECL_impl(name_))
+#define MPIR_T_PVAR_DOUBLE_LOWWATERMARK_DECL(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_LOWWATERMARK_DECL_impl(name_))
+
+#define MPIR_T_PVAR_UINT_LOWWATERMARK_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_LOWWATERMARK_DECL_STATIC_impl(name_))
+#define MPIR_T_PVAR_ULONG_LOWWATERMARK_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_LOWWATERMARK_DECL_STATIC_impl(name_))
+#define MPIR_T_PVAR_ULONG2_LOWWATERMARK_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_LOWWATERMARK_DECL_STATIC_impl(name_))
+#define MPIR_T_PVAR_DOUBLE_LOWWATERMARK_DECL_STATIC(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_LOWWATERMARK_DECL_STATIC_impl(name_))
+
+#define MPIR_T_PVAR_UINT_LOWWATERMARK_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_UINT_LOWWATERMARK_DECL_EXTERN_impl(name_))
+#define MPIR_T_PVAR_ULONG_LOWWATERMARK_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG_LOWWATERMARK_DECL_EXTERN_impl(name_))
+#define MPIR_T_PVAR_ULONG2_LOWWATERMARK_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_ULONG2_LOWWATERMARK_DECL_EXTERN_impl(name_))
+#define MPIR_T_PVAR_DOUBLE_LOWWATERMARK_DECL_EXTERN(MODULE, name_) \
+    PVAR_GATED_DECL(MODULE, MPIR_T_PVAR_DOUBLE_LOWWATERMARK_DECL_EXTERN_impl(name_))
+
+#define MPIR_T_PVAR_UINT_LOWWATERMARK_INIT_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_UINT_LOWWATERMARK_INIT_VAR_impl(ptr_, val_))
+#define MPIR_T_PVAR_ULONG_LOWWATERMARK_INIT_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG_LOWWATERMARK_INIT_VAR_impl(ptr_, val_))
+#define MPIR_T_PVAR_ULONG2_LOWWATERMARK_INIT_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG2_LOWWATERMARK_INIT_VAR_impl(ptr_, val_))
+#define MPIR_T_PVAR_DOUBLE_LOWWATERMARK_INIT_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_DOUBLE_LOWWATERMARK_INIT_VAR_impl(ptr_, val_))
+
+#define MPIR_T_PVAR_UINT_LOWWATERMARK_UPDATE_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_UINT_LOWWATERMARK_UPDATE_VAR_impl(ptr_, val_))
+#define MPIR_T_PVAR_ULONG_LOWWATERMARK_UPDATE_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG_LOWWATERMARK_UPDATE_VAR_impl(ptr_, val_))
+#define MPIR_T_PVAR_ULONG2_LOWWATERMARK_UPDATE_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG2_LOWWATERMARK_UPDATE_VAR_impl(ptr_, val_))
+#define MPIR_T_PVAR_DOUBLE_LOWWATERMARK_UPDATE_VAR(MODULE, ptr_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_DOUBLE_LOWWATERMARK_UPDATE_VAR_impl(ptr_, val_))
+
+#define MPIR_T_PVAR_UINT_LOWWATERMARK_INIT(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_UINT_LOWWATERMARK_INIT_impl(name_, val_))
+#define MPIR_T_PVAR_ULONG_LOWWATERMARK_INIT(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG_LOWWATERMARK_INIT_impl(name_, val_))
+#define MPIR_T_PVAR_ULONG2_LOWWATERMARK_INIT(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG2_LOWWATERMARK_INIT_impl(name_, val_))
+#define MPIR_T_PVAR_DOUBLE_LOWWATERMARK_INIT(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_DOUBLE_LOWWATERMARK_INIT_impl(name_, val_))
+
+#define MPIR_T_PVAR_UINT_LOWWATERMARK_UPDATE(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_UINT_LOWWATERMARK_UPDATE_impl(name_, val_))
+#define MPIR_T_PVAR_ULONG_LOWWATERMARK_UPDATE(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG_LOWWATERMARK_UPDATE_impl(name_, val_))
+#define MPIR_T_PVAR_ULONG2_LOWWATERMARK_UPDATE(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_ULONG2_LOWWATERMARK_UPDATE_impl(name_, val_))
+#define MPIR_T_PVAR_DOUBLE_LOWWATERMARK_UPDATE(MODULE, name_, val_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_DOUBLE_LOWWATERMARK_UPDATE_impl(name_, val_))
+
+#define MPIR_T_PVAR_LOWWATERMARK_REGISTER_STATIC(MODULE, dtype_, name_, \
+            initval_, verb_, bind_, flags_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_LOWWATERMARK_REGISTER_STATIC_impl(dtype_, name_, \
+            initval_, verb_, bind_, flags_, cat_, desc_))
+
+#define MPIR_T_PVAR_LOWWATERMARK_REGISTER_DYNAMIC(MODULE, dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \
+    PVAR_GATED_ACTION(MODULE, MPIR_T_PVAR_LOWWATERMARK_REGISTER_DYNAMIC_impl(dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_))
+
+#endif
diff --git a/src/include/mpitimpl.h b/src/include/mpitimpl.h
new file mode 100644
index 0000000..f84acd9
--- /dev/null
+++ b/src/include/mpitimpl.h
@@ -0,0 +1,1404 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
+/*
+ *  (C) 2013 by Argonne National Laboratory.
+ *      See COPYRIGHT in top-level directory.
+ */
+
+/* Types and interfaces in this file are internally used by MPIR_T itself.
+ * Other modules should call higher level interfaces in mpit.h instead.
+ */
+#ifndef MPIT_INTERNAL_H_INCLUDED
+#define MPIT_INTERNAL_H_INCLUDED
+
+#include "mpi.h"
+#include "mpiu_utarray.h"
+#include "mpiu_uthash.h"
+#include "mpiimplthread.h" /* For MPICH_IS_THREADED */
+
+/* MPI_T enum
+ */
+typedef struct enum_item_s {
+    const char* name;
+    int value;
+} enum_item_t;
+
+typedef struct MPIR_T_enum_s {
+    const char *name;
+    UT_array *items;
+} MPIR_T_enum_t;
+
+/* MPI_T category (cat)
+ */
+typedef struct {
+    const char *name;
+    UT_array *cvar_indices;
+    UT_array *pvar_indices;
+    UT_array *subcat_indices;
+    const char *desc;
+} cat_table_entry_t;
+
+/* Hash names to indices in a table */
+typedef struct {
+    const char *name;
+    int idx;
+    UT_hash_handle hh;  /* Makes this structure hashable */
+} name2index_hash_t;
+
+/* MPI_T control variable (cvar)
+ */
+typedef struct MPIR_T_cvar_range_value_s {
+    int low;
+    int high;
+} MPIR_T_cvar_range_value_t;
+
+/* Type used to represent cvar default values */
+typedef union MPIR_T_cvar_value_s {
+    int d;
+    unsigned u;
+    unsigned ul;
+    unsigned ull;
+    MPI_Count c;
+    char* str;
+    double f;
+    MPIR_T_cvar_range_value_t range;
+} MPIR_T_cvar_value_t;
+
+typedef int MPIR_T_cvar_get_addr_cb(void *obj_handle, void **addr);
+typedef int MPIR_T_cvar_get_count_cb(void *obj_handle, int *count);
+
+typedef struct cvar_table_entry_s {
+    /* Is the cvar currently in use? False if the cvar is unregistered */
+    int active;
+
+    /* cvar name */
+    const char *name;
+
+    /* Address of the cvar. May be NULL when get_addr != NULL */
+    void *addr;
+
+    /* cvar data type */
+    MPI_Datatype datatype;
+
+     /* Num. of elements of the cvar. May be ignored when get_count != NULL */
+    int count;
+
+    /* Properties of the cvar */
+    MPI_T_enum  enumtype;
+    MPIR_T_verbosity_t verbosity;
+    MPIR_T_bind_t bind;
+    MPIR_T_scope_t scope;
+
+    /* Default value */
+    MPIR_T_cvar_value_t defaultval;
+
+    /* If not NULL, components provide this callback to get addr of the cvar */
+    MPIR_T_cvar_get_addr_cb *get_addr;
+
+    /* If not NULL, components provide this callback to get count */
+    MPIR_T_cvar_get_count_cb *get_count;
+
+    /* Description of the cvar */
+    const char *desc;
+} cvar_table_entry_t;
+
+typedef struct MPIR_T_cvar_handle_s {
+     /* Address and count of the cvar. Set at handle allocation time */
+    void *addr;
+    int count;
+
+    /* Cached value from cvar_table_entry_t to avoid indirection */
+    MPI_Datatype datatype;
+    MPIR_T_scope_t scope;
+} MPIR_T_cvar_handle_t;
+
+void MPIR_T_CVAR_REGISTER_impl(
+    MPI_Datatype dtype, const char* name, const void *addr, int count,
+    MPIR_T_enum_t *etype, MPIR_T_verbosity_t verb, MPIR_T_bind_t bind,
+    MPIR_T_scope_t scope,MPIR_T_cvar_get_addr_cb get_addr,
+    MPIR_T_cvar_get_count_cb get_count, MPIR_T_cvar_value_t defaultval,
+    const char *cat, const char * desc);
+
+/* MPI_T performance variable (pvar)
+ */
+
+/* Forward declaration */
+struct MPIR_T_pvar_handle_s;
+struct MPIR_T_pvar_session_s;
+
+typedef int MPIR_T_pvar_get_value_cb(void *addr, void *obj_handle, int count, void *buf);
+typedef int MPIR_T_pvar_get_count_cb(void *addr, void *obj_handle, int *count);
+
+/* Basic pvar flags defined by MPI_T standard */
+#define MPIR_T_PVAR_FLAG_READONLY      0x01
+#define MPIR_T_PVAR_FLAG_CONTINUOUS    0x02
+#define MPIR_T_PVAR_FLAG_ATOMIC        0x04
+
+/* Auxlilary flags used by MPIR_T */
+
+/* pvar is MPI_T_PVAR_CLASS_{COUNTER, TIMER, AGGREGATE} */
+#define MPIR_T_PVAR_FLAG_SUM           0x08
+
+/* pvar is MPI_T_PVAR_CLASS_{HIGH, LOW}WATERMARK */
+#define MPIR_T_PVAR_FLAG_WATERMARK     0x10
+
+/* pvar is continuous. If not, it has been started at least once */
+#define MPIR_T_PVAR_FLAG_ONCESTARTED   0x20
+
+/* pvar is continuous. If not, it is started */
+#define MPIR_T_PVAR_FLAG_STARTED       0x40
+
+/* Used only for watermark handles. Set if a pvar handle is the
+ * first handle of an associated watermark.
+ */
+#define MPIR_T_PVAR_FLAG_FIRST         0x80
+
+/* MPI_T performance variable (pvar) stuff */
+typedef struct {
+    /* Is the pvar in use (i.e., not unregistered)? */
+    int active;
+
+    /* pvar name */
+    const char *name;
+
+    /* If not NULL, it is address of the pvar */
+    void *addr;
+
+    /* pvar data type */
+    MPI_Datatype datatype;
+
+     /* Num. of elements of the pvar */
+    int count;
+
+    /* Properties of the pvar */
+    MPIR_T_pvar_class_t varclass;
+    MPIR_T_verbosity_t verbosity;
+    MPIR_T_enum_t  *enumtype;
+    MPIR_T_bind_t bind;
+
+    /* Basic flags of the pvar */
+    int flags;
+
+    /* If not NULL, components provide this callback to read the pvar */
+    MPIR_T_pvar_get_value_cb *get_value;
+
+    /* If not NULL, components provide this callback to get count */
+    MPIR_T_pvar_get_count_cb *get_count;
+
+    /* Description of the pvar */
+    const char *desc;
+} pvar_table_entry_t;
+
+/*
+ The following two macros do not work since C preprocessor does not support
+ nested ifdefs. So we use another woarkable but a little ugly approach.
+
+#define PVAR_GATED_ACTION(MODULE, action_) \
+    do { \
+        #ifdef ENABLE_PVAR_##MODULE \
+            action_; \
+        #endif \
+    } while (0)
+
+#define PVAR_GATED_DECL(MODULE, declaration_) \
+    #ifdef ENABLE_PVAR_##MODULE \
+        declaration_; \
+    #endif
+*/
+
+/* ENABLE_PVAR_##MODULE must be defined by configure script either to 0 or 1 */
+#define PVAR_GATED_ACTION(MODULE, action_) \
+    do { \
+        if (ENABLE_PVAR_##MODULE) { \
+            action_; \
+        } \
+    } while (0)
+
+/* The declaration is not gated. But any compiler should optimize it away if
+ * there are no references to the declared varaible */
+#define PVAR_GATED_DECL(MODULE_, declaration_) \
+    declaration_
+
+/* For some classes of pvars, internally we can not represent them
+ * in basic data types. So come the following typedefs.
+ */
+
+/* Timer type */
+typedef struct {
+    MPID_Time_t total;
+    MPID_Time_t curstart;
+} MPIR_T_pvar_timer_t;
+
+/* An union to represent a watermark value */
+typedef union {
+    double f;
+    unsigned u;
+    unsigned long ul;
+    unsigned long long ull;
+} watermark_value_t;
+
+/* Watermark type */
+typedef struct {
+    /* current -- current resource utilization level
+     * waterarmk -- cached value for the first pvar handle
+     */
+    watermark_value_t current, watermark;
+
+    /* Datatype of the watermark */
+    MPI_Datatype datatype;
+
+    /* Is the cached value (i.e, watermark) in use by a pvar handle? */
+    int first_used;
+
+    /* Is the first pvar handle started? */
+    int first_started;
+
+    /* A double-linked list of handles of the pvar */
+    struct MPIR_T_pvar_handle_s *hlist;
+} MPIR_T_pvar_watermark_t;
+
+typedef struct MPIR_T_pvar_handle_s {
+    /* These are cached fields from pvar table. Do so to avoid extra
+     * indirection when accessing them through pvar handles.
+     */
+    void *addr;
+    MPI_Datatype datatype;
+    int count;
+    MPIR_T_pvar_get_value_cb *get_value;
+    MPIR_T_pvar_class_t varclass;
+
+    /* Bytes of an element of datatype */
+    int bytes;
+
+    /* Basic flags copied from pvar info + auxilary flags in pvar handle */
+    int flags;
+
+    /* Store info here in case we need other fields */
+    const pvar_table_entry_t *info;
+
+    /* Owner session from which the handle is allocated */
+    struct MPIR_T_pvar_session_s *session;
+
+    /* Object which this pvar is bound to. NULL if no binding */
+    void *obj_handle;
+
+    /* This is how we support pvar sessions.
+     *
+     * For pvars of counter, timer or aggregate type, we cache their value at
+     * the last start time in offset, their current value in current, and
+     * their accumlated value in accum. Generally, when such a pvar is running,
+     * reading the pvar should return
+     *      accum[i] + current[i] - offset[i], 0 <= i < count - 1.
+     * When the pvar is stopped, reading just returns accum.
+     *
+     * For pvars of high/lowwatermark type, above method does not work.
+     * We have a copy of such a pvar in every handle of the pvar.
+     * Handles are registered to the pvar. Whenever a watermark changes,
+     * its copies in non-stopped handles are updated. That sounds non-scalable.
+     * Considering single-session is common, we reserve room in watermark
+     * themselves for cache buffer for the first handle. So when such a pvar
+     * changes, it also updates the watermark close to it in memory.
+     *
+     * For pvars of other classes,  since they are supposed to be readonly
+     * and continuous (FIXME: Is it true?), caching is not needed.
+     */
+    void *accum;
+    void *offset;
+    void *current;
+
+    watermark_value_t watermark;
+
+    /* To chain handles in a session */
+    struct MPIR_T_pvar_handle_s *prev, *next;
+
+    /* To chain handles of a watermark pvar */
+    struct MPIR_T_pvar_handle_s *prev2, *next2;
+} MPIR_T_pvar_handle_t;
+
+typedef struct MPIR_T_pvar_session_s {
+    /* A linked list of pvar handles */
+    MPIR_T_pvar_handle_t *hlist;
+} MPIR_T_pvar_session_t;
+
+extern void MPIR_T_PVAR_REGISTER_impl(
+    int varclass, MPI_Datatype dtype, const char* name, void *addr, int count,
+    MPIR_T_enum_t *etype, int verb, int bind, int flags,
+    MPIR_T_pvar_get_value_cb get_value, MPIR_T_pvar_get_count_cb get_count,
+    const char * cat, const char * desc);
+
+/* For static pvars (i.e., pvars with static storage), we embed their class name
+ * into their variable name, so that users can declare pvars with the same name
+ * for different classes, without worry of name conflict. "class + pvar name"
+ * should be unique as required by MPI_T.
+ */
+
+/* MPI_T_PVAR_CLASS_STATE (continuous only)
+ */
+
+/* Declaration -- static pvar */
+#define MPIR_T_PVAR_INT_STATE_DECL_impl(name_) \
+    int PVAR_STATE_##name_;
+#define MPIR_T_PVAR_INT_STATE_DECL_STATIC_impl(name_) \
+    static int PVAR_STATE_##name_;
+#define MPIR_T_PVAR_INT_STATE_DECL_EXTERN_impl(name_) \
+    extern int PVAR_STATE_##name_;
+
+/* Interfaces through pointer or name */
+#define MPIR_T_PVAR_STATE_SET_VAR_impl(ptr_, val_) \
+    do { *ptr_ = val_; } while (0)
+#define MPIR_T_PVAR_STATE_GET_VAR_impl(ptr_) \
+    *(ptr_)
+
+#define MPIR_T_PVAR_STATE_SET_impl(name_, val_) \
+    MPIR_T_PVAR_STATE_SET_VAR_impl(&PVAR_STATE_##name_, val_)
+#define MPIR_T_PVAR_STATE_GET_impl(name_) \
+    MPIR_T_PVAR_STATE_GET_VAR_impl(&PVAR_STATE_##name_)
+
+/* Registration AND initialization for static pvar */
+#define MPIR_T_PVAR_STATE_REGISTER_STATIC_impl(dtype_, name_, \
+            initval_, etype_, verb_, bind_, flags_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes only */ \
+        MPIU_Assert(dtype_ == MPI_INT); \
+        /* Double check if dtype_ and name_ match */ \
+        MPIU_Assert(sizeof(PVAR_STATE_##name_) == MPID_Datatype_get_basic_size(dtype_)); \
+        MPIU_Assert(flags_ & MPIR_T_PVAR_FLAG_CONTINUOUS); \
+        /* State pvars should be describled further by an enum */ \
+        MPIU_Assert(etype_ != MPI_T_ENUM_NULL); \
+        PVAR_STATE_##name_ = initval_; \
+        void *addr_ = &PVAR_STATE_##name_; \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_STATE, dtype_, #name_, \
+            addr_, 1, etype_, verb_, bind_, flags_, NULL, NULL, cat_, desc_); \
+    } while (0)
+
+/* Registration for dynamic pvar w/ or w/o callback. Init is left to users */
+#define MPIR_T_PVAR_STATE_REGISTER_DYNAMIC_impl(dtype_, name_, addr_, count_, \
+            etype_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes */ \
+        MPIU_Assert(dtype_ == MPI_INT); \
+        MPIU_Assert(flags_ & MPIR_T_PVAR_FLAG_CONTINUOUS); \
+        MPIU_Assert(etype_ != MPI_T_ENUM_NULL); \
+        MPIU_Assert(addr_ != NULL || get_value_ != NULL); \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_STATE, dtype_, #name_, \
+            addr_, count_, etype_, verb_, bind_, flags_, get_value_, cat_, desc_); \
+    } while (0)
+
+
+/* MPI_T_PVAR_CLASS_LEVEL (continuous only)
+ */
+
+/* Declaration -- static pvar */
+#define MPIR_T_PVAR_UINT_LEVEL_DECL_impl(name_) \
+    unsigned int PVAR_LEVEL_##name_;
+#define MPIR_T_PVAR_ULONG_LEVEL_DECL_impl(name_) \
+    unsigned long PVAR_LEVEL_##name_;
+#define MPIR_T_PVAR_ULONG2_LEVEL_DECL_impl(name_) \
+    unsigned long long PVAR_LEVEL_##name_;
+#define MPIR_T_PVAR_DOUBLE_LEVEL_DECL_impl(name_) \
+    double PVAR_LEVEL_##name_;
+
+#define MPIR_T_PVAR_UINT_LEVEL_DECL_STATIC_impl(name_) \
+    static unsigned PVAR_LEVEL_##name_;
+#define MPIR_T_PVAR_ULONG_LEVEL_DECL_STATIC_impl(name_) \
+    static unsigned long PVAR_LEVEL_##name_;
+#define MPIR_T_PVAR_ULONG2_LEVEL_DECL_STATIC_impl(name_) \
+    static unsigned long long PVAR_LEVEL_##name_;
+#define MPIR_T_PVAR_DOUBLE_LEVEL_DECL_STATIC_impl(name_) \
+    static double PVAR_LEVEL_##name_;
+
+#define MPIR_T_PVAR_UINT_LEVEL_DECL_EXTERN_impl(name_) \
+    extern unsigned PVAR_LEVEL_##name_;
+#define MPIR_T_PVAR_ULONG_LEVEL_DECL_EXTERN_impl(name_) \
+    extern unsigned long PVAR_LEVEL_##name_;
+#define MPIR_T_PVAR_ULONG2_LEVEL_DECL_EXTERN_impl(name_) \
+    extern unsigned long long PVAR_LEVEL_##name_;
+#define MPIR_T_PVAR_DOUBLE_LEVEL_DECL_EXTERN_impl(name_) \
+    extern double PVAR_LEVEL_##name_;
+
+/* Interfaces through pointer or name */
+#define MPIR_T_PVAR_LEVEL_SET_VAR_impl(ptr_, val_) \
+    do { *(ptr_) = val_; } while (0)
+#define MPIR_T_PVAR_LEVEL_GET_VAR_impl(ptr_) \
+    *(ptr)
+#define MPIR_T_PVAR_LEVEL_INC_VAR_impl(ptr_, val_) \
+    do { *(ptr_) += val_; } while (0)
+#define MPIR_T_PVAR_LEVEL_DEC_VAR_impl(ptr_, val_) \
+    do { *(ptr_) -= val_; } while (0)
+
+#define MPIR_T_PVAR_LEVEL_SET_impl(name_, val_) \
+    MPIR_T_PVAR_LEVEL_SET_VAR_impl(&PVAR_LEVEL_##name_, val_)
+#define MPIR_T_PVAR_LEVEL_GET_impl(name_) \
+    MPIR_T_PVAR_LEVEL_GET_VAR_impl(&PVAR_LEVEL_##name_)
+#define MPIR_T_PVAR_LEVEL_INC_impl(name_, val_) \
+    MPIR_T_PVAR_LEVEL_INC_VAR_impl(&PVAR_LEVEL_##name_, val_)
+#define MPIR_T_PVAR_LEVEL_DEC_impl(name_, val_) \
+    MPIR_T_PVAR_LEVEL_DEC_VAR_impl(&PVAR_LEVEL_##name_, val_)
+
+/* Registration AND initialization for static pvar */
+#define MPIR_T_PVAR_LEVEL_REGISTER_STATIC_impl(dtype_, name_, \
+            initval_, verb_, bind_, flags_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes only */ \
+        MPIU_Assert(dtype_ == MPI_UNSIGNED || dtype_ == MPI_UNSIGNED_LONG || \
+                    dtype_ == MPI_UNSIGNED_LONG_LONG || dtype_ == MPI_DOUBLE); \
+        /* Double check if dtype_ and name_ match */ \
+        MPIU_Assert(sizeof(PVAR_LEVEL_##name_) == MPID_Datatype_get_basic_size(dtype_)); \
+        MPIU_Assert(flags_ & MPIR_T_PVAR_FLAG_CONTINUOUS); \
+        PVAR_LEVEL_##name_ = initval_; \
+        void *addr_ = &PVAR_LEVEL_##name_; \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_LEVEL, dtype_, #name_, \
+            addr_, 1, MPI_T_ENUM_NULL, verb_, bind_, flags_, NULL, NULL, cat_, desc_); \
+    } while (0)
+
+/* Registration for dynamic pvar w/ or w/o callback. Init is left to users */
+#define MPIR_T_PVAR_LEVEL_REGISTER_DYNAMIC_impl(dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count, cat_, desc_) \
+    do { \
+        /* Allowable datatypes */ \
+        MPIU_Assert(dtype_ == MPI_UNSIGNED || dtype_ == MPI_UNSIGNED_LONG || \
+                    dtype_ == MPI_UNSIGNED_LONG_LONG || dtype_ == MPI_DOUBLE); \
+        MPIU_Assert(flags_ & MPIR_T_PVAR_FLAG_CONTINUOUS); \
+        MPIU_Assert(addr_ != NULL || get_value_ != NULL); \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_LEVEL, dtype_, #name_, \
+            addr_, count_, MPI_T_ENUM_NULL, verb_, bind_, flags_, get_value_, \
+            get_count, cat_, desc_); \
+    } while (0)
+
+
+/* MPI_T_PVAR_CLASS_SIZE (continuous only)
+ */
+
+/* Declaration -- static pvar */
+#define MPIR_T_PVAR_UINT_SIZE_DECL_impl(name_) \
+    unsigned PVAR_SIZE_##name_;
+#define MPIR_T_PVAR_ULONG_SIZE_DECL_impl(name_) \
+    unsigned long PVAR_SIZE_##name_;
+#define MPIR_T_PVAR_ULONG2_SIZE_DECL_impl(name_) \
+    unsigned long long PVAR_SIZE_##name_;
+#define MPIR_T_PVAR_DOUBLE_SIZE_DECL_impl(name_) \
+    double PVAR_SIZE_##name_;
+
+#define MPIR_T_PVAR_UINT_SIZE_DECL_STATIC_impl(name_) \
+    static unsigned PVAR_SIZE_##name_;
+#define MPIR_T_PVAR_ULONG_SIZE_DECL_STATIC_impl(name_) \
+    static unsigned long PVAR_SIZE_##name_;
+#define MPIR_T_PVAR_ULONG2_SIZE_DECL_STATIC_impl(name_) \
+    static unsigned long long PVAR_SIZE_##name_;
+#define MPIR_T_PVAR_DOUBLE_SIZE_DECL_STATIC_impl(name_) \
+    static double PVAR_SIZE_##name_;
+
+#define MPIR_T_PVAR_UINT_SIZE_DECL_EXTERN_impl(name_) \
+    extern unsigned PVAR_SIZE_##name_;
+#define MPIR_T_PVAR_ULONG_SIZE_DECL_EXTERN_impl(name_) \
+    extern unsigned long PVAR_SIZE_##name_;
+#define MPIR_T_PVAR_ULONG2_SIZE_DECL_EXTERN_impl(name_) \
+    extern unsigned long long PVAR_SIZE_##name_;
+#define MPIR_T_PVAR_DOUBLE_SIZE_DECL_EXTERN_impl(name_) \
+    extern double PVAR_SIZE_##name_;
+
+/* Interfaces through pointer or name */
+#define MPIR_T_PVAR_SIZE_SET_VAR_impl(ptr_, val_) \
+    do { *(ptr_) = val_; } while (0)
+#define MPIR_T_PVAR_SIZE_GET_VAR_impl(ptr_) \
+    *(ptr_)
+
+#define MPIR_T_PVAR_SIZE_SET_impl(name_, val_) \
+    MPIR_T_PVAR_SIZE_SET_VAR_impl(&PVAR_SIZE_##name_, val_)
+#define MPIR_T_PVAR_SIZE_GET_impl(name_) \
+    MPIR_T_PVAR_SIZE_GET_VAR_impl(&PVAR_SIZE_##name_)
+
+/* Registration AND initialization for static pvar */
+#define MPIR_T_PVAR_SIZE_REGISTER_STATIC_impl(dtype_, name_, \
+            initval_, verb_, bind_, flags_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes only */ \
+        MPIU_Assert(dtype_ == MPI_UNSIGNED || dtype_ == MPI_UNSIGNED_LONG || \
+                    dtype_ == MPI_UNSIGNED_LONG_LONG || dtype_ == MPI_DOUBLE); \
+        /* Double check if dtype_ and name_ match */ \
+        MPIU_Assert(sizeof(PVAR_SIZE_##name_) == MPID_Datatype_get_basic_size(dtype_)); \
+        MPIU_Assert(flags_ & MPIR_T_PVAR_FLAG_CONTINUOUS); \
+        PVAR_SIZE_##name_ = initval_; \
+        void *addr_ = &PVAR_SIZE_##name_; \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_SIZE, dtype_, #name_, \
+            addr_, 1, MPI_T_ENUM_NULL, verb_, bind_, flags_, NULL, NULL, cat_, desc_); \
+    } while (0)
+
+/* Registration for dynamic pvar w/ or w/o callback. Init is left to users */
+#define MPIR_T_PVAR_SIZE_REGISTER_DYNAMIC_impl(dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes */ \
+        MPIU_Assert(dtype_ == MPI_UNSIGNED || dtype_ == MPI_UNSIGNED_LONG || \
+                    dtype_ == MPI_UNSIGNED_LONG_LONG || dtype_ == MPI_DOUBLE); \
+        MPIU_Assert(flags_ & MPIR_T_PVAR_FLAG_CONTINUOUS); \
+        MPIU_Assert(addr_ != NULL || get_value_ != NULL); \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_SIZE, dtype_, #name_, \
+            addr_, count_, MPI_T_ENUM_NULL, verb_, bind_, flags_, get_value_, \
+            get_count_, cat_, desc_); \
+    } while (0)
+
+
+/* MPI_T_PVAR_CLASS_PERCENTAGE (continuous only)
+ */
+
+/* Declaration -- static pvar */
+#define MPIR_T_PVAR_DOUBLE_PERCENTAGE_DECL_impl(name_) \
+    double PVAR_PERCENTAGE_##name_;
+
+#define MPIR_T_PVAR_DOUBLE_PERCENTAGE_DECL_STATIC_impl(name_) \
+    static double PVAR_PERCENTAGE_##name_;
+
+#define MPIR_T_PVAR_DOUBLE_PERCENTAGE_DECL_EXTERN_impl(name_) \
+    extern double PVAR_PERCENTAGE_##name_;
+
+/* Interfaces through pointer or name */
+#define MPIR_T_PVAR_PERCENTAGE_SET_VAR_impl(ptr_, val_) \
+    do { \
+        MPIU_Assert(0.0 <= val_ && val_ <= 1.0); \
+        *ptr_ = val_; \
+    } while (0)
+#define MPIR_T_PVAR_PERCENTAGE_GET_VAR_impl(ptr_) \
+    *(ptr_)
+
+#define MPIR_T_PVAR_PERCENTAGE_SET_impl(name_, val_) \
+    MPIR_T_PVAR_PERCENTAGE_SET_VAR_impl(&PVAR_PERCENTAGE_##name_, val_)
+#define MPIR_T_PVAR_PERCENTAGE_GET_impl(name_) \
+    MPIR_T_PVAR_PERCENTAGE_GET_VAR_impl(&PVAR_PERCENTAGE_##name_)
+
+/* Registration AND initialization for static pvar */
+#define MPIR_T_PVAR_PERCENTAGE_REGISTER_STATIC_impl(dtype_, name_, \
+            initval_, verb_, bind_, flags_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes only */ \
+        MPIU_Assert(dtype_ == MPI_DOUBLE); \
+        /* Double check if dtype_ and name_ match */ \
+        MPIU_Assert(sizeof(PVAR_PERCENTAGE_##name_) == MPID_Datatype_get_basic_size(dtype_)); \
+        MPIU_Assert(flags_ & MPIR_T_PVAR_FLAG_CONTINUOUS); \
+        void *addr_ = &PVAR_PERCENTAGE_##name_; \
+        PVAR_PERCENTAGE_##name_ = initval_; \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_PERCENTAGE, dtype_, #name_, \
+            addr_, 1, MPI_T_ENUM_NULL, verb_, bind_, flags_, NULL, NULL, cat_, desc_); \
+    } while (0)
+
+/* Registration for dynamic pvar w/ or w/o callback. Init is left to users */
+#define MPIR_T_PVAR_PERCENTAGE_REGISTER_DYNAMIC_impl(dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes */ \
+        MPIU_Assert(dtype_ == MPI_DOUBLE); \
+        MPIU_Assert(flags_ & MPIR_T_PVAR_FLAG_CONTINUOUS); \
+        MPIU_Assert(addr_ != NULL || get_value_ != NULL); \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_PERCENTAGE, dtype_, #name_, \
+            addr_, count_, MPI_T_ENUM_NULL, verb_, bind_, flags_, get_value_, \
+            get_count_, cat_, desc_); \
+    } while (0)
+
+
+/* MPI_T_PVAR_CLASS_COUNTER (continuous or not)
+ */
+
+/* Declaration -- static pvar */
+#define MPIR_T_PVAR_UINT_COUNTER_DECL_impl(name_) \
+    unsigned PVAR_COUNTER_##name_;
+#define MPIR_T_PVAR_ULONG_COUNTER_DECL_impl(name_) \
+    unsigned long PVAR_COUNTER_##name_;
+#define MPIR_T_PVAR_ULONG2_COUNTER_DECL_impl(name_) \
+    unsigned long long PVAR_COUNTER_##name_;
+
+#define MPIR_T_PVAR_UINT_COUNTER_DECL_STATIC_impl(name_) \
+    static unsigned PVAR_COUNTER_##name_;
+#define MPIR_T_PVAR_ULONG_COUNTER_DECL_STATIC_impl(name_) \
+    static unsigned long PVAR_COUNTER_##name_;
+#define MPIR_T_PVAR_ULONG2_COUNTER_DECL_STATIC_impl(name_) \
+    static unsigned long long PVAR_COUNTER_##name_;
+
+#define MPIR_T_PVAR_UINT_COUNTER_DECL_EXTERN_impl(name_) \
+    extern unsigned PVAR_COUNTER_##name_;
+#define MPIR_T_PVAR_ULONG_COUNTER_DECL_EXTERN_impl(name_) \
+    extern unsigned long PVAR_COUNTER_##name_;
+#define MPIR_T_PVAR_ULONG2_COUNTER_DECL_EXTERN_impl(name_) \
+    extern unsigned long long PVAR_COUNTER_##name_;
+
+/* Interfaces through pointer or name */
+#define MPIR_T_PVAR_COUNTER_INIT_VAR_impl(ptr_) \
+    do { *(ptr_) = 0; } while (0)
+/* _COUNTER_SET is intentionally not provided. Users should only INC counters */
+#define MPIR_T_PVAR_COUNTER_GET_VAR_impl(ptr_) \
+    *(ptr_)
+#define MPIR_T_PVAR_COUNTER_INC_VAR_impl(ptr_, inc_) \
+    do { *(ptr_) += inc_; } while (0)
+
+#define MPIR_T_PVAR_COUNTER_INIT_impl(name_) \
+    MPIR_T_PVAR_COUNTER_INIT_VAR_impl(&PVAR_COUNTER_##name_)
+#define MPIR_T_PVAR_COUNTER_GET_impl(name_) \
+    MPIR_T_PVAR_COUNTER_GET_VAR_impl(&PVAR_COUNTER_##name_)
+#define MPIR_T_PVAR_COUNTER_INC_impl(name_, inc_) \
+    MPIR_T_PVAR_COUNTER_INC_VAR_impl(&PVAR_COUNTER_##name_, inc_)
+
+/* Registration AND initialization to zero for static pvar.  */
+#define MPIR_T_PVAR_COUNTER_REGISTER_STATIC_impl(dtype_, name_, \
+            verb_, bind_, flags_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes only */ \
+        MPIU_Assert(dtype_ == MPI_UNSIGNED || dtype_ == MPI_UNSIGNED_LONG || \
+                    dtype_ == MPI_UNSIGNED_LONG_LONG); \
+        /* Double check if dtype_ and name_ match*/ \
+        MPIU_Assert(sizeof(PVAR_COUNTER_##name_) == MPID_Datatype_get_basic_size(dtype_)); \
+        PVAR_COUNTER_##name_ = 0; \
+        void *addr_ = &PVAR_COUNTER_##name_; \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_COUNTER, dtype_, #name_, \
+            addr_, 1, MPI_T_ENUM_NULL, verb_, bind_, flags_, NULL, NULL, cat_, desc_); \
+    } while (0)
+
+/* Registration for dynamic pvar w/ or w/o callback. Init is left to users */
+#define MPIR_T_PVAR_COUNTER_REGISTER_DYNAMIC_impl(dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes */ \
+        MPIU_Assert(dtype_ == MPI_UNSIGNED || dtype_ == MPI_UNSIGNED_LONG || \
+                    dtype_ == MPI_UNSIGNED_LONG_LONG); \
+        MPIU_Assert(addr_ != NULL || get_value_ != NULL); \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_COUNTER, dtype_, #name_, \
+            addr_, count_, MPI_T_ENUM_NULL, verb_, bind_, flags_, get_value_, \
+            get_count_, cat_, desc_); \
+    } while (0)
+
+/* Declaration of a counter array -- static pvar */
+#define MPIR_T_PVAR_UINT_COUNTER_ARRAY_DECL_impl(name_, len_) \
+    unsigned PVAR_COUNTER_##name_[len_];
+#define MPIR_T_PVAR_ULONG_COUNTER_ARRAY_DECL_impl(name_, len_) \
+    unsigned long PVAR_COUNTER_##name_[len_];
+#define MPIR_T_PVAR_ULONG2_COUNTER_ARRAY_DECL_impl(name_, len_) \
+    unsigned long long PVAR_COUNTER_##name_[len_];
+
+#define MPIR_T_PVAR_UINT_COUNTER_ARRAY_DECL_STATIC_impl(name_, len_) \
+    static unsigned PVAR_COUNTER_##name_[len_];
+#define MPIR_T_PVAR_ULONG_COUNTER_ARRAY_DECL_STATIC_impl(name_, len_) \
+    static unsigned long PVAR_COUNTER_##name_[len_];
+#define MPIR_T_PVAR_ULONG2_COUNTER_ARRAY_DECL_STATIC_impl(name_, len_) \
+    static unsigned long long PVAR_COUNTER_##name_[len_];
+
+#define MPIR_T_PVAR_UINT_COUNTER_ARRAY_DECL_EXTERN_impl(name_, len_) \
+    extern unsigned PVAR_COUNTER_##name_[len_];
+#define MPIR_T_PVAR_ULONG_COUNTER_ARRAY_DECL_EXTERN_impl(name_, len_) \
+    extern unsigned long PVAR_COUNTER_##name_[len_];
+#define MPIR_T_PVAR_ULONG2_COUNTER_ARRAY_DECL_EXTERN_impl(name_, len_) \
+    extern unsigned long long PVAR_COUNTER_##name_[len_];
+
+/* Interfaces through pointer or name */
+#define MPIR_T_PVAR_COUNTER_ARRAY_INIT_VAR_impl(ptr_, count_) \
+    do { \
+        int idx_ = 0; \
+        for (; idx_ < count_; idx_++) \
+            *(ptr_ + idx_) = 0; \
+    } while (0)
+#define MPIR_T_PVAR_COUNTER_ARRAY_GET_VAR_impl(ptr_, idx_) \
+    *(ptr_ + idx_)
+#define MPIR_T_PVAR_COUNTER_ARRAY_INC_VAR_impl(ptr_, idx_, inc_) \
+    do { *(ptr_ + idx_) += inc_; } while (0)
+
+#define MPIR_T_PVAR_COUNTER_ARRAY_INIT_impl(name_) \
+    do { \
+        int count_ = sizeof(PVAR_COUNTER_##name_)/sizeof(PVAR_COUNTER_##name_[0]); \
+        MPIR_T_PVAR_COUNTER_ARRAY_INIT_VAR_impl(PVAR_COUNTER_##name_, count_); \
+    } while (0)
+#define MPIR_T_PVAR_COUNTER_ARRAY_GET_impl(name_, idx_) \
+    MPIR_T_PVAR_COUNTER_ARRAY_GET_VAR_impl(PVAR_COUNTER_##name_, idx_)
+#define MPIR_T_PVAR_COUNTER_ARRAY_INC_impl(ptr_, idx_, inc_) \
+    MPIR_T_PVAR_COUNTER_ARRAY_INC_VAR_impl(PVAR_COUNTER_##name_, idx_, inc_)
+
+/* Registration AND initialization to zero for static counter array  */
+#define MPIR_T_PVAR_COUNTER_ARRAY_REGISTER_STATIC_impl(dtype_, name_, \
+            verb_, bind_, flags_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes only */ \
+        MPIU_Assert(dtype_ == MPI_UNSIGNED || dtype_ == MPI_UNSIGNED_LONG || \
+                    dtype_ == MPI_UNSIGNED_LONG_LONG); \
+        /* Double check if dtype_ and name_ match */ \
+        MPIU_Assert(sizeof(PVAR_COUNTER_##name_[0]) == MPID_Datatype_get_basic_size(dtype_)); \
+        void *addr_ = PVAR_COUNTER_##name_; \
+        MPIR_T_PVAR_COUNTER_ARRAY_INIT_impl(name_); \
+        int count_ = sizeof(PVAR_COUNTER_##name_)/sizeof(mpit_pvar_##name_[0]); \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_COUNTER, dtype_, #name_, \
+            addr_, count_, MPI_T_ENUM_NULL, verb_, bind_, flags_, NULL, NULL, cat_, desc_); \
+    } while (0)
+
+/* Dynamic counter array is already provided by MPIR_T_PVAR_COUNTER_REGISTER_DYNAMIC */
+
+/* MPI_T_PVAR_CLASS_AGGREGATE (continuous or not)
+ */
+
+/* Declaration -- static aggregate */
+#define MPIR_T_PVAR_UINT_AGGREGATE_DECL_impl(name_) \
+    unsigned PVAR_AGGREGATE_##name_;
+#define MPIR_T_PVAR_ULONG_AGGREGATE_DECL_impl(name_) \
+    unsigned long PVAR_AGGREGATE_##name_;
+#define MPIR_T_PVAR_ULONG2_AGGREGATE_DECL_impl(name_) \
+    unsigned long long PVAR_AGGREGATE_##name_;
+#define MPIR_T_PVAR_DOUBLE_AGGREGATE_DECL_impl(name_) \
+    double PVAR_AGGREGATE_##name_;
+
+#define MPIR_T_PVAR_UINT_AGGREGATE_DECL_STATIC_impl(name_) \
+    static unsigned PVAR_AGGREGATE_##name_;
+#define MPIR_T_PVAR_ULONG_AGGREGATE_DECL_STATIC_impl(name_) \
+    static unsigned long PVAR_AGGREGATE_##name_;
+#define MPIR_T_PVAR_ULONG2_AGGREGATE_DECL_STATIC_impl(name_) \
+    static unsigned long long PVAR_AGGREGATE_##name_;
+#define MPIR_T_PVAR_DOUBLE_AGGREGATE_DECL_STATIC_impl(name_) \
+    static double PVAR_AGGREGATE_##name_;
+
+#define MPIR_T_PVAR_UINT_AGGREGATE_DECL_EXTERN_impl(name_) \
+    extern unsigned PVAR_AGGREGATE_##name_;
+#define MPIR_T_PVAR_ULONG_AGGREGATE_DECL_EXTERN_impl(name_) \
+    extern unsigned long PVAR_AGGREGATE_##name_;
+#define MPIR_T_PVAR_ULONG2_AGGREGATE_DECL_EXTERN_impl(name_) \
+    extern unsigned long long PVAR_AGGREGATE_##name_;
+#define MPIR_T_PVAR_DOUBLE_AGGREGATE_DECL_EXTERN_impl(name_) \
+    extern double PVAR_AGGREGATE_##name_;
+
+/* Interfaces through pointer or name */
+#define MPIR_T_PVAR_AGGREGATE_INIT_VAR_impl(ptr_) \
+    do { *(ptr_) = 0; } while (0)
+/* _AGGREGATE_SET is intentionally not provided as for counters */
+#define MPIR_T_PVAR_AGGREGATE_GET_VAR_impl(ptr_) \
+    *(ptr_)
+#define MPIR_T_PVAR_AGGREGATE_INC_VAR_impl(ptr_, inc_) \
+    do { *(ptr_) += inc_; } while (0)
+
+#define MPIR_T_PVAR_AGGREGATE_INIT_impl(name_) \
+    MPIR_T_PVAR_AGGREGATE_INIT_VAR_impl(&PVAR_AGGREGATE_##name_)
+#define MPIR_T_PVAR_AGGREGATE_GET_impl(name_) \
+    MPIR_T_PVAR_AGGREGATE_GET_VAR_impl(&PVAR_AGGREGATE_##name_)
+#define MPIR_T_PVAR_AGGREGATE_INC_impl(name_, inc_) \
+    MPIR_T_PVAR_AGGREGATE_INC_VAR_impl(&PVAR_AGGREGATE_##name_, inc_)
+
+/* Registration AND initialization to zero for static aggregate  */
+#define MPIR_T_PVAR_AGGREGATE_REGISTER_STATIC_impl(dtype_, name_, \
+            verb_, bind_, flags_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes only */ \
+        MPIU_Assert(dtype_ == MPI_UNSIGNED || dtype_ == MPI_UNSIGNED_LONG || \
+                    dtype_ == MPI_UNSIGNED_LONG_LONG || dtype_ == MPI_DOUBLE); \
+        /* Double check if dtype_ and name_ match*/ \
+        MPIU_Assert(sizeof(PVAR_AGGREGATE_##name_) == MPID_Datatype_get_basic_size(dtype_)); \
+        PVAR_AGGREGATE_##name_ = 0; \
+        void *addr_ = &PVAR_AGGREGATE_##name_; \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_AGGREGATE, dtype_, #name_, \
+            addr_, 1, MPI_T_ENUM_NULL, verb_, bind_, flags_, NULL, NULL, cat_, desc_); \
+    } while (0)
+
+/* Registration for dynamic pvar w/ or w/o callback. Init is left to users */
+#define MPIR_T_PVAR_AGGREGATE_REGISTER_DYNAMIC_impl(dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes */ \
+        MPIU_Assert(dtype_ == MPI_UNSIGNED || dtype_ == MPI_UNSIGNED_LONG || \
+                    dtype_ == MPI_UNSIGNED_LONG_LONG || dtype_ == MPI_DOUBLE); \
+        MPIU_Assert(addr_ != NULL || get_value_ != NULL); \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_AGGREGATE, dtype_, #name_, \
+            addr_, count_, MPI_T_ENUM_NULL, verb_, bind_, flags_, get_value_, \
+            get_count_, cat_, desc_); \
+    } while (0)
+
+
+/* MPI_T_PVAR_CLASS_TIMER (continuous or not)
+ */
+
+/* Declaration -- static timer */
+#define MPIR_T_PVAR_DOUBLE_TIMER_DECL_impl(name_) \
+    MPIR_T_pvar_timer_t PVAR_TIMER_##name_;
+
+#define MPIR_T_PVAR_DOUBLE_TIMER_DECL_STATIC_impl(name_) \
+    static MPIR_T_pvar_timer_t PVAR_TIMER_##name_;
+
+#define MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN_impl(name_) \
+    extern MPIR_T_pvar_timer_t PVAR_TIMER_##name_;
+
+/* Interfaces through pointer or name */
+#define MPIR_T_PVAR_TIMER_INIT_VAR_impl(ptr_) \
+    do { \
+        /* FIXME: need a generic approach to init a timer */ \
+        memset(&((ptr_)->total), 0, sizeof(MPID_Time_t)); \
+    } while (0)
+#define MPIR_T_PVAR_TIMER_GET_VAR_impl(ptr_, buf) \
+    do { \
+        MPID_Wtime_todouble(&((ptr_)->total), buf); \
+    } while (0)
+#define MPIR_T_PVAR_TIMER_START_VAR_impl(ptr_) \
+    do { \
+        MPID_Wtime(&((ptr_)->curstart)); \
+    } while (0)
+#define MPIR_T_PVAR_TIMER_END_VAR_impl(ptr_) \
+    do { \
+        MPID_Time_t tmp; \
+        MPID_Wtime(&tmp); \
+        MPID_Wtime_acc(&((ptr_)->curstart), &tmp, &((ptr_)->total)); \
+    } while (0)
+
+#define MPIR_T_PVAR_TIMER_INIT_impl(name_) \
+    MPIR_T_PVAR_TIMER_INIT_VAR_impl(&PVAR_TIMER_##name_)
+#define MPIR_T_PVAR_TIMER_GET_impl(name_, buf_) \
+    MPIR_T_PVAR_TIMER_GET_VAR_impl(&PVAR_TIMER_##name_, buf_)
+#define MPIR_T_PVAR_TIMER_START_impl(name_) \
+    MPIR_T_PVAR_TIMER_START_VAR_impl(&PVAR_TIMER_##name_)
+#define MPIR_T_PVAR_TIMER_END_impl(name_) \
+    MPIR_T_PVAR_TIMER_END_VAR_impl(&PVAR_TIMER_##name_)
+
+/* Customized get_value() for MPIR_T_pvar_timer_t */
+static inline
+int get_timer_in_double(MPIR_T_pvar_timer_t *timer, void *obj_handle,
+                    int count, double *buf)
+{
+    int i, mpi_errno = MPI_SUCCESS;
+
+    for (i = 0; i < count; i++)
+        MPID_Wtime_todouble(&(timer[i].total), &buf[i]);
+
+    return mpi_errno;
+}
+
+/* Registration for static storage */
+#define MPIR_T_PVAR_TIMER_REGISTER_STATIC_impl(dtype_, name_, \
+            verb_, bind_, flags_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes only */ \
+        MPIU_Assert(dtype_ == MPI_DOUBLE); \
+        MPIR_T_PVAR_TIMER_INIT_impl(name_); \
+        void *addr_ = &PVAR_TIMER_##name_; \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_TIMER, dtype_, #name_, \
+            addr_, 1, MPI_T_ENUM_NULL, verb_, bind_, flags_, \
+            (MPIR_T_pvar_get_value_cb *)&get_timer_in_double, NULL, cat_, desc_); \
+    } while (0)
+
+/* Registration for dynamic pvar w/ or w/o callback. Init is left to users.
+ * (Probably, the callback still is get_timer_in_double)
+ */
+#define MPIR_T_PVAR_TIMER_REGISTER_DYNAMIC_impl(dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes only */ \
+        MPIU_Assert(dtype_ == MPI_DOUBLE); \
+        MPIU_Assert(addr_ != NULL || get_value_ != NULL); \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_TIMER, dtype_, #name_, \
+            addr_, count_, MPI_T_ENUM_NULL, verb_, bind_, flags_, \
+            get_value_, get_count_, cat_, desc_); \
+    } while (0)
+
+
+/* MPI_T_PVAR_CLASS_HIGHWATERMARK (continuous or not)
+ */
+
+/* Declaration -- static pvar */
+#define MPIR_T_PVAR_UINT_HIGHWATERMARK_DECL_impl(name_) \
+    MPIR_T_pvar_watermark_t PVAR_HIGHWATERMARK_##name_;
+#define MPIR_T_PVAR_ULONG_HIGHWATERMARK_DECL_impl(name_) \
+    MPIR_T_pvar_watermark_t PVAR_HIGHWATERMARK_##name_;
+#define MPIR_T_PVAR_ULONG2_HIGHWATERMARK_DECL_impl(name_) \
+    MPIR_T_pvar_watermark_t PVAR_HIGHWATERMARK_##name_;
+#define MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_DECL_impl(name_) \
+    MPIR_T_pvar_watermark_t PVAR_HIGHWATERMARK_##name_;
+
+#define MPIR_T_PVAR_UINT_HIGHWATERMARK_DECL_STATIC_impl(name_) \
+    static MPIR_T_pvar_watermark_t PVAR_HIGHWATERMARK_##name_;
+#define MPIR_T_PVAR_ULONG_HIGHWATERMARK_DECL_STATIC_impl(name_) \
+    static MPIR_T_pvar_watermark_t PVAR_HIGHWATERMARK_##name_;
+#define MPIR_T_PVAR_ULONG2_HIGHWATERMARK_DECL_STATIC_impl(name_) \
+    static MPIR_T_pvar_watermark_t PVAR_HIGHWATERMARK_##name_;
+#define MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_DECL_STATIC_impl(name_) \
+    static MPIR_T_pvar_watermark_t PVAR_HIGHWATERMARK_##name_;
+
+#define MPIR_T_PVAR_UINT_HIGHWATERMARK_DECL_EXTERN_impl(name_) \
+    extern MPIR_T_pvar_watermark_t PVAR_HIGHWATERMARK_##name_;
+#define MPIR_T_PVAR_ULONG_HIGHWATERMARK_DECL_EXTERN_impl(name_) \
+    extern MPIR_T_pvar_watermark_t PVAR_HIGHWATERMARK_##name_;
+#define MPIR_T_PVAR_ULONG2_HIGHWATERMARK_DECL_EXTERN_impl(name_) \
+    extern MPIR_T_pvar_watermark_t PVAR_HIGHWATERMARK_##name_;
+#define MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_DECL_EXTERN_impl(name_) \
+    extern MPIR_T_pvar_watermark_t PVAR_HIGHWATERMARK_##name_;
+
+/* Interfaces through pointer or name.
+ * In contrast to previous pvar classes, for each type we create a set
+ * of interfaces. That is because we have a pointer and a union in the
+ * struct. We need to know types to (de)reference them.
+*/
+#define MPIR_T_PVAR_UINT_HIGHWATERMARK_INIT_VAR_impl(ptr_, val_) \
+    do { \
+        (ptr_)->datatype = MPI_UNSIGNED; \
+        (ptr_)->current.u = val_; \
+        (ptr_)->first_started = 0;    \
+        (ptr_)->first_used = 0; \
+        (ptr_)->hlist = NULL;  \
+    } while (0)
+
+#define MPIR_T_PVAR_ULONG_HIGHWATERMARK_INIT_VAR_impl(ptr_, val_) \
+    do { \
+        (ptr_)->datatype = MPI_UNSIGNED_LONG; \
+        (ptr_)->current.ul = val_; \
+        (ptr_)->first_started = 0;    \
+        (ptr_)->first_used = 0; \
+        (ptr_)->hlist = NULL;  \
+    } while (0)
+
+#define MPIR_T_PVAR_ULONG2_HIGHWATERMARK_INIT_VAR_impl(ptr_, val_) \
+    do { \
+        (ptr_)->datatype = MPI_UNSIGNED_LONG_LONG; \
+        (ptr_)->current.ull = val_; \
+        (ptr_)->first_started = 0;    \
+        (ptr_)->first_used = 0; \
+        (ptr_)->hlist = NULL;  \
+    } while (0)
+
+#define MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_INIT_VAR_impl(ptr_, val_) \
+    do { \
+        (ptr_)->datatype = MPI_DOUBLE; \
+        (ptr_)->current.f = val_; \
+        (ptr_)->first_started = 0;    \
+        (ptr_)->first_used = 0; \
+        (ptr_)->hlist = NULL;  \
+    } while (0)
+
+#define MPIR_T_PVAR_UINT_HIGHWATERMARK_UPDATE_VAR_impl(ptr_, val_) \
+    do { \
+        (ptr_)->current.u = val_; \
+        if ((ptr_)->first_used && (ptr_)->first_started) { \
+            if ((val_) > (ptr_)->watermark.u) \
+                (ptr_)->watermark.u = val_; \
+        } \
+        MPIR_T_pvar_handle_t *head = (ptr_)->hlist; \
+        while (head != NULL) { \
+            if (MPIR_T_pvar_is_started(head) && (val_) > head->watermark.u) { \
+                head->watermark.u = val_; \
+            } \
+            head = head->next2; \
+        } \
+    } while (0)
+
+#define MPIR_T_PVAR_ULONG_HIGHWATERMARK_UPDATE_VAR_impl(ptr_, val_) \
+    do { \
+        (ptr_)->current.ul = val_; \
+        if ((ptr_)->first_used && (ptr_)->first_started) { \
+            if ((val_) > (ptr_)->watermark.ul) \
+                (ptr_)->watermark.ul = val_; \
+        } \
+        MPIR_T_pvar_handle_t *head = (ptr_)->hlist; \
+        while (head != NULL) { \
+            if (MPIR_T_pvar_is_started(head) && (val_) > head->watermark.ul) { \
+                head->watermark.ul = val_; \
+            } \
+            head = head->next2; \
+        } \
+    } while (0)
+
+#define MPIR_T_PVAR_ULONG2_HIGHWATERMARK_UPDATE_VAR_impl(ptr_, val_) \
+    do { \
+        (ptr_)->current.ull = val_; \
+        if ((ptr_)->first_used && (ptr_)->first_started) { \
+            if ((val_) > (ptr_)->watermark.ull) \
+                (ptr_)->watermark.ull = val_; \
+        } \
+        MPIR_T_pvar_handle_t *head = (ptr_)->hlist; \
+        while (head != NULL) { \
+            if (MPIR_T_pvar_is_started(head) && (val_) > head->watermark.ull) { \
+                head->watermark.ull = val_; \
+            } \
+            head = head->next2; \
+        } \
+    } while (0)
+
+#define MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_UPDATE_VAR_impl(ptr_, val_) \
+    do { \
+        (ptr_)->current.f = val_; \
+        if ((ptr_)->first_used && (ptr_)->first_started) { \
+            if ((val_) > (ptr_)->watermark.f) \
+                (ptr_)->watermark.f = val_; \
+        } \
+        MPIR_T_pvar_handle_t *head = (ptr_)->hlist; \
+        while (head != NULL) { \
+            if (MPIR_T_pvar_is_started(head) && (val_) > head->watermark.f) { \
+                head->watermark.f = val_; \
+            } \
+            head = head->next2; \
+        } \
+    } while (0)
+
+#define MPIR_T_PVAR_UINT_HIGHWATERMARK_INIT_impl(name_, val_) \
+    MPIR_T_PVAR_UINT_HIGHWATERMARK_INIT_VAR_impl(&PVAR_HIGHWATERMARK_##name_, val_)
+#define MPIR_T_PVAR_ULONG_HIGHWATERMARK_INIT_impl(name_, val_) \
+    MPIR_T_PVAR_ULONG_HIGHWATERMARK_INIT_VAR_impl(&PVAR_HIGHWATERMARK_##name_, val_)
+#define MPIR_T_PVAR_ULONG2_HIGHWATERMARK_INIT_impl(name_, val_) \
+    MPIR_T_PVAR_ULONG2_HIGHWATERMARK_INIT_VAR_impl(&PVAR_HIGHWATERMARK_##name_, val_)
+#define MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_INIT_impl(name_, val_) \
+    MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_INIT_VAR_impl(&PVAR_HIGHWATERMARK_##name_, val_)
+
+#define MPIR_T_PVAR_UINT_HIGHWATERMARK_UPDATE_impl(name_, val_) \
+    MPIR_T_PVAR_UINT_HIGHWATERMARK_UPDATE_VAR_impl(&PVAR_HIGHWATERMARK_##name_, val_)
+#define MPIR_T_PVAR_ULONG_HIGHWATERMARK_UPDATE_impl(name_, val_) \
+    MPIR_T_PVAR_ULONG_HIGHWATERMARK_UPDATE_VAR_impl(&PVAR_HIGHWATERMARK_##name_, val_)
+#define MPIR_T_PVAR_ULONG2_HIGHWATERMARK_UPDATE_impl(name_, val_) \
+    MPIR_T_PVAR_ULONG2_HIGHWATERMARK_UPDATE_VAR_impl(&PVAR_HIGHWATERMARK_##name_, val_)
+#define MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_UPDATE_impl(name_, val_) \
+    MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_UPDATE_VAR_impl(&PVAR_HIGHWATERMARK_##name_, val_)
+
+/* Registration AND initialization for static pvar  */
+#define MPIR_T_PVAR_HIGHWATERMARK_REGISTER_STATIC_impl(dtype_, name_, \
+            initval_, verb_, bind_, flags_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes only */ \
+        MPIU_Assert(dtype_ == MPI_UNSIGNED || dtype_ == MPI_UNSIGNED_LONG || \
+                    dtype_ == MPI_UNSIGNED_LONG_LONG || dtype_ == MPI_DOUBLE); \
+        switch (dtype_) { \
+        case MPI_UNSIGNED: \
+            MPIR_T_PVAR_UINT_HIGHWATERMARK_INIT_impl(name_, initval_); break; \
+        case MPI_UNSIGNED_LONG: \
+            MPIR_T_PVAR_ULONG_HIGHWATERMARK_INIT_impl(name_, initval_); break; \
+        case MPI_UNSIGNED_LONG_LONG: \
+            MPIR_T_PVAR_ULONG2_HIGHWATERMARK_INIT_impl(name_, initval_); break; \
+        case MPI_DOUBLE: \
+            MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_INIT_impl(name_, initval_); break; \
+        default: \
+            break; \
+        }; \
+        void *addr_ = &PVAR_HIGHWATERMARK_##name_; \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_HIGHWATERMARK, dtype_, #name_, \
+            addr_, 1, MPI_T_ENUM_NULL, verb_, bind_, flags_, NULL, cat_, desc_); \
+    } while (0)
+
+/* Registration for dynamic pvar w/ or w/o callback. Init is left to users */
+#define MPIR_T_PVAR_HIGHWATERMARK_REGISTER_DYNAMIC_impl(dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes */ \
+        MPIU_Assert(dtype_ == MPI_UNSIGNED || dtype_ == MPI_UNSIGNED_LONG || \
+                    dtype_ == MPI_UNSIGNED_LONG_LONG || dtype_ == MPI_DOUBLE); \
+        MPIU_Assert(addr_ != NULL || get_value_ != NULL); \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_HIGHWATERMARK, dtype_, #name_, \
+            addr_, count_, MPI_T_ENUM_NULL, verb_, bind_, flags_, get_value_, \
+            get_count_, cat_, desc_); \
+    } while (0)
+
+
+/* MPI_T_PVAR_CLASS_LOWWATERMARK (continuous or not)
+ */
+
+/* Declaration -- static pvar */
+#define MPIR_T_PVAR_UINT_LOWWATERMARK_DECL_impl(name_) \
+    MPIR_T_pvar_watermark_t PVAR_LOWWATERMARK_##name_;
+#define MPIR_T_PVAR_ULONG_LOWWATERMARK_DECL_impl(name_) \
+    MPIR_T_pvar_watermark_t PVAR_LOWWATERMARK_##name_;
+#define MPIR_T_PVAR_ULONG2_LOWWATERMARK_DECL_impl(name_) \
+    MPIR_T_pvar_watermark_t PVAR_LOWWATERMARK_##name_;
+#define MPIR_T_PVAR_DOUBLE_LOWWATERMARK_DECL_impl(name_) \
+    MPIR_T_pvar_watermark_t PVAR_LOWWATERMARK_##name_;
+
+#define MPIR_T_PVAR_UINT_LOWWATERMARK_DECL_STATIC_impl(name_) \
+    static MPIR_T_pvar_watermark_t PVAR_LOWWATERMARK_##name_;
+#define MPIR_T_PVAR_ULONG_LOWWATERMARK_DECL_STATIC_impl(name_) \
+    static MPIR_T_pvar_watermark_t PVAR_LOWWATERMARK_##name_;
+#define MPIR_T_PVAR_ULONG2_LOWWATERMARK_DECL_STATIC_impl(name_) \
+    static MPIR_T_pvar_watermark_t PVAR_LOWWATERMARK_##name_;
+#define MPIR_T_PVAR_DOUBLE_LOWWATERMARK_DECL_STATIC_impl(name_) \
+    static MPIR_T_pvar_watermark_t PVAR_LOWWATERMARK_##name_;
+
+#define MPIR_T_PVAR_UINT_LOWWATERMARK_DECL_EXTERN_impl(name_) \
+    extern MPIR_T_pvar_watermark_t PVAR_LOWWATERMARK_##name_;
+#define MPIR_T_PVAR_ULONG_LOWWATERMARK_DECL_EXTERN_impl(name_) \
+    extern MPIR_T_pvar_watermark_t PVAR_LOWWATERMARK_##name_;
+#define MPIR_T_PVAR_ULONG2_LOWWATERMARK_DECL_EXTERN_impl(name_) \
+    extern MPIR_T_pvar_watermark_t PVAR_LOWWATERMARK_##name_;
+#define MPIR_T_PVAR_DOUBLE_LOWWATERMARK_DECL_EXTERN_impl(name_) \
+    extern MPIR_T_pvar_watermark_t PVAR_LOWWATERMARK_##name_;
+
+#define MPIR_T_PVAR_UINT_LOWWATERMARK_INIT_VAR_impl(ptr_, val_) \
+    do { \
+        (ptr_)->datatype = MPI_UNSIGNED; \
+        (ptr_)->current.u = val_; \
+        (ptr_)->first_started = 0;    \
+        (ptr_)->first_used = 0; \
+        (ptr_)->hlist = NULL;  \
+    } while (0)
+
+#define MPIR_T_PVAR_ULONG_LOWWATERMARK_INIT_VAR_impl(ptr_, val_) \
+    do { \
+        (ptr_)->datatype = MPI_UNSIGNED_LONG; \
+        (ptr_)->current.ul = val_; \
+        (ptr_)->first_started = 0;    \
+        (ptr_)->first_used = 0; \
+        (ptr_)->hlist = NULL;  \
+    } while (0)
+
+#define MPIR_T_PVAR_ULONG2_LOWWATERMARK_INIT_VAR_impl(ptr_, val_) \
+    do { \
+        (ptr_)->datatype = MPI_UNSIGNED_LONG_LONG; \
+        (ptr_)->current.ull = val_; \
+        (ptr_)->first_started = 0;    \
+        (ptr_)->first_used = 0; \
+        (ptr_)->hlist = NULL;  \
+    } while (0)
+
+#define MPIR_T_PVAR_DOUBLE_LOWWATERMARK_INIT_VAR_impl(ptr_, val_) \
+    do { \
+        (ptr_)->datatype = MPI_DOUBLE; \
+        (ptr_)->current.f = val_; \
+        (ptr_)->first_started = 0;    \
+        (ptr_)->first_used = 0; \
+        (ptr_)->hlist = NULL;  \
+    } while (0)
+
+#define MPIR_T_PVAR_UINT_LOWWATERMARK_UPDATE_VAR_impl(ptr_, val_) \
+    do { \
+        (ptr_)->current.u = val_; \
+        /* Update values in all handles */ \
+        if ((ptr_)->first_used && (ptr_)->first_started) { \
+            if ((val_) < (ptr_)->watermark.u) \
+                (ptr_)->watermark.u = val_; \
+        } \
+        MPIR_T_pvar_handle_t *head = (ptr_)->hlist; \
+        while (head != NULL) { \
+            if (MPIR_T_pvar_is_started(head) && (val_) < head->watermark.u) { \
+                head->watermark.u = val_; \
+            } \
+            head = head->next2; \
+        } \
+    } while (0)
+
+#define MPIR_T_PVAR_ULONG_LOWWATERMARK_UPDATE_VAR_impl(ptr_, val_) \
+    do { \
+        (ptr_)->current.ul = val_; \
+        if ((ptr_)->first_used && (ptr_)->first_started) { \
+            if ((val_) < (ptr_)->watermark.ul) \
+                (ptr_)->watermark.ul = val_; \
+        } \
+        MPIR_T_pvar_handle_t *head = (ptr_)->hlist; \
+        while (head != NULL) { \
+            if (MPIR_T_pvar_is_started(head) && (val_) < head->watermark.ul) { \
+                head->watermark.ul = val_; \
+            } \
+            head = head->next2; \
+        } \
+    } while (0)
+
+#define MPIR_T_PVAR_ULONG2_LOWWATERMARK_UPDATE_VAR_impl(ptr_, val_) \
+    do { \
+        (ptr_)->current.ull = val_; \
+        if ((ptr_)->first_used && (ptr_)->first_started) { \
+            if ((val_) < (ptr_)->watermark.ull) \
+                (ptr_)->watermark.ull = val_; \
+        } \
+        MPIR_T_pvar_handle_t *head = (ptr_)->hlist; \
+        while (head != NULL) { \
+            if (MPIR_T_pvar_is_started(head) && (val_) < head->watermark.ull) { \
+                head->watermark.ull = val_; \
+            } \
+            head = head->next2; \
+        } \
+    } while (0)
+
+#define MPIR_T_PVAR_DOUBLE_LOWWATERMARK_UPDATE_VAR_impl(ptr_, val_) \
+    do { \
+        (ptr_)->current.f = val_; \
+        if ((ptr_)->first_used && (ptr_)->first_started) { \
+            if ((val_) < (ptr_)->watermark.f) \
+                (ptr_)->watermark.f = val_; \
+        } \
+        MPIR_T_pvar_handle_t *head = (ptr_)->hlist; \
+        while (head != NULL) { \
+            if (MPIR_T_pvar_is_started(head) && (val_) < head->watermark.f) { \
+                head->watermark.f = val_; \
+            } \
+            head = head->next2; \
+        } \
+    } while (0)
+
+#define MPIR_T_PVAR_UINT_LOWWATERMARK_INIT_impl(name_, val_) \
+    MPIR_T_PVAR_UINT_LOWWATERMARK_INIT_VAR_impl(&PVAR_LOWWATERMARK_##name_, val_)
+#define MPIR_T_PVAR_ULONG_LOWWATERMARK_INIT_impl(name_, val_) \
+    MPIR_T_PVAR_ULONG_LOWWATERMARK_INIT_VAR_impl(&PVAR_LOWWATERMARK_##name_, val_)
+#define MPIR_T_PVAR_ULONG2_LOWWATERMARK_INIT_impl(name_, val_) \
+    MPIR_T_PVAR_ULONG2_LOWWATERMARK_INIT_VAR_impl(&PVAR_LOWWATERMARK_##name_, val_)
+#define MPIR_T_PVAR_DOUBLE_LOWWATERMARK_INIT_impl(name_, val_) \
+    MPIR_T_PVAR_DOUBLE_LOWWATERMARK_INIT_VAR_impl(&PVAR_LOWWATERMARK_##name_, val_)
+
+#define MPIR_T_PVAR_UINT_LOWWATERMARK_UPDATE_impl(name_, val_) \
+    MPIR_T_PVAR_UINT_LOWWATERMARK_UPDATE_VAR_impl(&PVAR_LOWWATERMARK_##name_, val_)
+#define MPIR_T_PVAR_ULONG_LOWWATERMARK_UPDATE_impl(name_, val_) \
+    MPIR_T_PVAR_ULONG_LOWWATERMARK_UPDATE_VAR_impl(&PVAR_LOWWATERMARK_##name_, val_)
+#define MPIR_T_PVAR_ULONG2_LOWWATERMARK_UPDATE_impl(name_, val_) \
+    MPIR_T_PVAR_ULONG2_LOWWATERMARK_UPDATE_VAR_impl(&PVAR_LOWWATERMARK_##name_, val_)
+#define MPIR_T_PVAR_DOUBLE_LOWWATERMARK_UPDATE_impl(name_, val_) \
+    MPIR_T_PVAR_DOUBLE_LOWWATERMARK_UPDATE_VAR_impl(&PVAR_LOWWATERMARK_##name_, val_)
+
+/* Registration AND initialization for static pvar  */
+#define MPIR_T_PVAR_LOWWATERMARK_REGISTER_STATIC_impl(dtype_, name_, \
+            initval_, verb_, bind_, flags_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes only */ \
+        MPIU_Assert(dtype_ == MPI_UNSIGNED || dtype_ == MPI_UNSIGNED_LONG || \
+                    dtype_ == MPI_UNSIGNED_LONG_LONG || dtype_ == MPI_DOUBLE); \
+        switch (dtype_) { \
+        case MPI_UNSIGNED: \
+            MPIR_T_PVAR_UINT_LOWWATERMARK_INIT_impl(name_, initval_); break; \
+        case MPI_UNSIGNED_LONG: \
+            MPIR_T_PVAR_ULONG_LOWWATERMARK_INIT_impl(name_, initval_); break; \
+        case MPI_UNSIGNED_LONG_LONG: \
+            MPIR_T_PVAR_ULONG2_LOWWATERMARK_INIT_impl(name_, initval_); break; \
+        case MPI_DOUBLE: \
+            MPIR_T_PVAR_DOUBLE_LOWWATERMARK_INIT_impl(name_, initval_); break; \
+        default: \
+            break; \
+        }; \
+        void *addr_ = &PVAR_LOWWATERMARK_##name_; \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_LOWWATERMARK, dtype_, #name_, \
+            addr_, 1, MPI_T_ENUM_NULL, verb_, bind_, flags_, NULL, cat_, desc_); \
+    } while (0)
+
+/* Registration for dynamic pvar w/ or w/o callback. Init is left to users */
+#define MPIR_T_PVAR_LOWWATERMARK_REGISTER_DYNAMIC_impl(dtype_, name_, \
+            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \
+    do { \
+        /* Allowable datatypes */ \
+        MPIU_Assert(dtype_ == MPI_UNSIGNED || dtype_ == MPI_UNSIGNED_LONG || \
+                    dtype_ == MPI_UNSIGNED_LONG_LONG || dtype_ == MPI_DOUBLE); \
+        MPIU_Assert(addr_ != NULL || get_value_ != NULL); \
+        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_LOWWATERMARK, dtype_, #name_, \
+            addr_, count_, MPI_T_ENUM_NULL, verb_, bind_, flags_, get_value_, \
+            get_count_, cat_, desc_); \
+    } while (0)
+
+/* Unregister a pvar by its index */
+#define MPIR_T_PVAR_UNREGISTER(idx_) \
+    do { \
+        if (pvar_table != NULL) { \
+            pvar_table_entry_t *pvar = \
+                (pvar_table_entry_t *)utarray_eltptr(pvar_table, idx_); \
+            if (pvar != NULL) { \
+                pvar->active = FALSE; \
+                /* Do not do MPIU_Free(pvar->info), since it may be re-activated */ \
+            } \
+        } \
+    } while (0)
+
+static inline int MPIR_T_pvar_is_readonly(MPIR_T_pvar_handle_t *handle) {
+    return handle->flags & MPIR_T_PVAR_FLAG_READONLY;
+}
+
+static inline int MPIR_T_pvar_is_continuous(MPIR_T_pvar_handle_t *handle) {
+    return handle->flags & MPIR_T_PVAR_FLAG_CONTINUOUS;
+}
+
+static inline int MPIR_T_pvar_is_atomic(MPIR_T_pvar_handle_t *handle) {
+    return handle->flags & MPIR_T_PVAR_FLAG_ATOMIC;
+}
+
+static inline int MPIR_T_pvar_is_sum(MPIR_T_pvar_handle_t *handle) {
+    return handle->flags & MPIR_T_PVAR_FLAG_SUM;
+}
+
+static inline int MPIR_T_pvar_is_watermark(MPIR_T_pvar_handle_t *handle) {
+    return handle->flags & MPIR_T_PVAR_FLAG_WATERMARK;
+}
+
+static inline int MPIR_T_pvar_is_started(MPIR_T_pvar_handle_t *handle) {
+    return handle->flags & MPIR_T_PVAR_FLAG_STARTED;
+}
+
+static inline void MPIR_T_pvar_set_started(MPIR_T_pvar_handle_t *handle) {
+    handle->flags |= (MPIR_T_PVAR_FLAG_STARTED | MPIR_T_PVAR_FLAG_ONCESTARTED);
+}
+
+static inline void MPIR_T_pvar_unset_started(MPIR_T_pvar_handle_t *handle) {
+    handle->flags &= ~MPIR_T_PVAR_FLAG_STARTED;
+}
+
+static inline int MPIR_T_pvar_is_oncestarted(MPIR_T_pvar_handle_t *handle) {
+    return handle->flags & MPIR_T_PVAR_FLAG_ONCESTARTED;
+}
+
+static inline void MPIR_T_pvar_unset_oncestarted(MPIR_T_pvar_handle_t *handle) {
+    handle->flags &= ~MPIR_T_PVAR_FLAG_ONCESTARTED;
+}
+
+static inline int MPIR_T_pvar_is_first(MPIR_T_pvar_handle_t *handle) {
+    return handle->flags & MPIR_T_PVAR_FLAG_FIRST;
+}
+
+static inline int MPIR_T_pvar_set_first(MPIR_T_pvar_handle_t *handle) {
+    return handle->flags |= MPIR_T_PVAR_FLAG_FIRST;
+}
+
+static inline int MPIR_T_pvar_unset_first(MPIR_T_pvar_handle_t *handle) {
+    return handle->flags &= ~MPIR_T_PVAR_FLAG_FIRST;
+}
+
+/* A counter that keeps track of the relative balance of calls to
+ * MPI_T_init_thread and MPI_T_finalize */
+extern int MPIR_T_init_balance;
+static inline int MPIR_T_is_initialized() {
+    return MPIR_T_init_balance > 0;
+}
+
+/* A special strncpy to return strings in behavior defined by MPI_T */
+extern void MPIR_T_strncpy(char *dst, const char *src, int *len);
+
+/* Stuffs to support multithreaded MPI_T */
+#ifdef HAVE_RUNTIME_THREADCHECK
+extern int MPIR_T_is_threaded;
+#define MPIR_T_THREAD_CHECK_BEGIN if (MPIR_T_is_threaded) {
+#define MPIR_T_THREAD_CHECK_END }
+#else /* !HAVE_RUNTIME_THREADCHECK */
+#define MPIR_T_THREAD_CHECK_BEGIN
+#define MPIR_T_THREAD_CHECK_END
+#endif
+
+#ifdef MPICH_IS_THREADED
+extern MPIU_Thread_mutex_t mpi_t_mutex;
+#define MPIR_T_THREAD_CS_INIT() \
+    do { \
+        int err_; \
+        MPIR_T_THREAD_CHECK_BEGIN \
+        MPID_Thread_mutex_create(&mpi_t_mutex, &err_); \
+        MPIU_Assert(err_ == 0); \
+        MPIR_T_THREAD_CHECK_END \
+    } while (0)
+
+#define MPIR_T_THREAD_CS_FINALIZE() \
+    do { \
+        int err_; \
+        MPIR_T_THREAD_CHECK_BEGIN \
+        MPID_Thread_mutex_destroy(&mpi_t_mutex, &err_); \
+        MPIU_Assert(err_ == 0); \
+        MPIR_T_THREAD_CHECK_END \
+    } while (0)
+
+#define MPIR_T_THREAD_CS_ENTER() \
+    do { \
+        MPIR_T_THREAD_CHECK_BEGIN \
+        MPID_Thread_mutex_lock(&mpi_t_mutex); \
+        MPIR_T_THREAD_CHECK_END \
+    } while (0)
+
+#define MPIR_T_THREAD_CS_EXIT() \
+    do { \
+        MPIR_T_THREAD_CHECK_BEGIN \
+        MPID_Thread_mutex_unlock(&mpi_t_mutex); \
+        MPIR_T_THREAD_CHECK_END \
+    } while (0)
+#else /* !MPICH_IS_THREADED */
+#define MPIR_T_THREAD_CS_INIT()     do { /* nothing */ } while (0)
+#define MPIR_T_THREAD_CS_FINALIZE() do { /* nothing */ } while (0)
+#define MPIR_T_THREAD_CS_ENTER()    do { /* nothing */ } while (0)
+#define MPIR_T_THREAD_CS_EXIT()     do { /* nothing */ } while (0)
+#endif
+
+/* Init and finalize routines */
+extern void MPIR_T_env_init();
+extern void MPIR_T_env_finalize();
+
+#define MPIR_T_FAIL_IF_UNINITIALIZED() \
+    do { \
+        if (!MPIR_T_is_initialized()) { \
+            mpi_errno = MPI_T_ERR_NOT_INITIALIZED; \
+            goto fn_fail; \
+        } \
+    } while (0)
+
+#endif
diff --git a/src/mpi_t/cat_changed.c b/src/mpi_t/cat_changed.c
index 989d102..aebff45 100644
--- a/src/mpi_t/cat_changed.c
+++ b/src/mpi_t/cat_changed.c
@@ -21,25 +21,6 @@
 #ifndef MPICH_MPI_FROM_PMPI
 #undef MPI_T_category_changed
 #define MPI_T_category_changed PMPI_T_category_changed
-
-/* any non-MPI functions go here, especially non-static ones */
-
-#undef FUNCNAME
-#define FUNCNAME MPIR_T_category_changed_impl
-#undef FCNAME
-#define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPIR_T_category_changed_impl(int *stamp)
-{
-    int mpi_errno = MPI_ERR_INTERN;
-
-    /* TODO implement this function */
-
-fn_exit:
-    return mpi_errno;
-fn_fail:
-    goto fn_exit;
-}
-
 #endif /* MPICH_MPI_FROM_PMPI */
 
 #undef FUNCNAME
@@ -61,48 +42,22 @@ Output Parameters:
 int MPI_T_category_changed(int *stamp)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CATEGORY_CHANGED);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CATEGORY_CHANGED);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_CATEGORY_CHANGED);
 
-    /* Validate parameters, especially handles needing to be converted */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            MPIR_ERRTEST_ARGNULL(stamp, "stamp", mpi_errno);
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
     /* ... body of routine ...  */
 
-    mpi_errno = MPIR_T_category_changed_impl(stamp);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    if (stamp != NULL)
+        *stamp = cat_stamp;
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_CATEGORY_CHANGED);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/cat_get_categories.c b/src/mpi_t/cat_get_categories.c
index 5f61d63..93e088a 100644
--- a/src/mpi_t/cat_get_categories.c
+++ b/src/mpi_t/cat_get_categories.c
@@ -30,9 +30,17 @@
 #define FCNAME MPIU_QUOTE(FUNCNAME)
 int MPIR_T_category_get_categories_impl(int cat_index, int len, int indices[])
 {
-    int mpi_errno = MPI_ERR_INTERN;
+    int mpi_errno = MPI_SUCCESS;
+    cat_table_entry_t *cat;
+    int i, num_subcats, count;
+
+    cat = (cat_table_entry_t *)utarray_eltptr(cat_table, cat_index);
+    num_subcats = utarray_len(cat->subcat_indices);
+    count = len < num_subcats ? len : num_subcats;
 
-    /* TODO implement this function */
+    for (i = 0; i < count; i++) {
+        indices[i] = *(int *)utarray_eltptr(cat->subcat_indices, i);
+    }
 
 fn_exit:
     return mpi_errno;
@@ -65,39 +73,31 @@ Output Parameters:
 int MPI_T_category_get_categories(int cat_index, int len, int indices[])
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CATEGORY_GET_CATEGORIES);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CATEGORY_GET_CATEGORIES);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_CATEGORY_GET_CATEGORIES);
 
-    /* Validate parameters, especially handles needing to be converted */
+    /* Validate parameters */
 #   ifdef HAVE_ERROR_CHECKING
     {
         MPID_BEGIN_ERROR_CHECKS
         {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            if (len != 0)
+                MPIR_ERRTEST_ARGNULL(indices, "indices", mpi_errno);
         }
         MPID_END_ERROR_CHECKS
     }
 #   endif /* HAVE_ERROR_CHECKING */
 
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
+    /* ... body of routine ...  */
+    if (cat_index < 0 || cat_index >= utarray_len(cat_table)) {
+        mpi_errno = MPI_T_ERR_INVALID_INDEX;
+        goto fn_fail;
     }
-#   endif /* HAVE_ERROR_CHECKING */
 
-    /* ... body of routine ...  */
+    if (len == 0) goto fn_exit;
 
     mpi_errno = MPIR_T_category_get_categories_impl(cat_index, len, indices);
     if (mpi_errno) MPIU_ERR_POP(mpi_errno);
@@ -106,7 +106,7 @@ int MPI_T_category_get_categories(int cat_index, int len, int indices[])
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_CATEGORY_GET_CATEGORIES);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/cat_get_cvars.c b/src/mpi_t/cat_get_cvars.c
index cfdf92c..5117c30 100644
--- a/src/mpi_t/cat_get_cvars.c
+++ b/src/mpi_t/cat_get_cvars.c
@@ -30,9 +30,17 @@
 #define FCNAME MPIU_QUOTE(FUNCNAME)
 int MPIR_T_category_get_cvars_impl(int cat_index, int len, int indices[])
 {
-    int mpi_errno = MPI_ERR_INTERN;
+    int mpi_errno = MPI_SUCCESS;
+    cat_table_entry_t *cat;
+    int i, num_cvars, count;
+
+    cat = (cat_table_entry_t *)utarray_eltptr(cat_table, cat_index);
+    num_cvars = utarray_len(cat->cvar_indices);
+    count = len < num_cvars ? len : num_cvars;
 
-    /* TODO implement this function */
+    for (i = 0; i < count; i++) {
+        indices[i] = *(int *)utarray_eltptr(cat->cvar_indices, i);
+    }
 
 fn_exit:
     return mpi_errno;
@@ -65,39 +73,31 @@ Output Parameters:
 int MPI_T_category_get_cvars(int cat_index, int len, int indices[])
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CATEGORY_GET_CVARS);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CATEGORY_GET_CVARS);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_CATEGORY_GET_CVARS);
 
-    /* Validate parameters, especially handles needing to be converted */
+    /* Validate parameters */
 #   ifdef HAVE_ERROR_CHECKING
     {
         MPID_BEGIN_ERROR_CHECKS
         {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            if (len != 0)
+                MPIR_ERRTEST_ARGNULL(indices, "indices", mpi_errno);
         }
         MPID_END_ERROR_CHECKS
     }
 #   endif /* HAVE_ERROR_CHECKING */
 
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
+    /* ... body of routine ...  */
+    if (cat_index < 0 || cat_index >= utarray_len(cat_table)) {
+        mpi_errno = MPI_T_ERR_INVALID_INDEX;
+        goto fn_fail;
     }
-#   endif /* HAVE_ERROR_CHECKING */
 
-    /* ... body of routine ...  */
+    if (len == 0) goto fn_exit;
 
     mpi_errno = MPIR_T_category_get_cvars_impl(cat_index, len, indices);
     if (mpi_errno) MPIU_ERR_POP(mpi_errno);
@@ -106,7 +106,7 @@ int MPI_T_category_get_cvars(int cat_index, int len, int indices[])
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_CATEGORY_GET_CVARS);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/cat_get_info.c b/src/mpi_t/cat_get_info.c
index 6f95643..4bdb274 100644
--- a/src/mpi_t/cat_get_info.c
+++ b/src/mpi_t/cat_get_info.c
@@ -21,25 +21,6 @@
 #ifndef MPICH_MPI_FROM_PMPI
 #undef MPI_T_category_get_info
 #define MPI_T_category_get_info PMPI_T_category_get_info
-
-/* any non-MPI functions go here, especially non-static ones */
-
-#undef FUNCNAME
-#define FUNCNAME MPIR_T_category_get_info_impl
-#undef FCNAME
-#define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPIR_T_category_get_info_impl(int cat_index, char *name, int *name_len, char *desc, int *desc_len, int *num_cvars, int *num_pvars, int *num_categories)
-{
-    int mpi_errno = MPI_ERR_INTERN;
-
-    /* TODO implement this function */
-
-fn_exit:
-    return mpi_errno;
-fn_fail:
-    goto fn_exit;
-}
-
 #endif /* MPICH_MPI_FROM_PMPI */
 
 #undef FUNCNAME
@@ -69,55 +50,53 @@ Output Parameters:
 
 .N Errors
 @*/
-int MPI_T_category_get_info(int cat_index, char *name, int *name_len, char *desc, int *desc_len, int *num_cvars, int *num_pvars, int *num_categories)
+int MPI_T_category_get_info(int cat_index, char *name, int *name_len, char *desc,
+        int *desc_len, int *num_cvars, int *num_pvars, int *num_categories)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CATEGORY_GET_INFO);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CATEGORY_GET_INFO);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_CATEGORY_GET_INFO);
 
-    /* Validate parameters, especially handles needing to be converted */
+    /* Validate parameters */
 #   ifdef HAVE_ERROR_CHECKING
     {
         MPID_BEGIN_ERROR_CHECKS
         {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            /* Do not do _TEST_ARGNULL for any argument, since this is
+             * allowed or will be allowed by MPI_T standard.
+             */
         }
         MPID_END_ERROR_CHECKS
     }
 #   endif /* HAVE_ERROR_CHECKING */
 
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            MPIR_ERRTEST_ARGNULL(name_len, "name_len", mpi_errno);
-            MPIR_ERRTEST_ARGNULL(desc_len, "desc_len", mpi_errno);
-            MPIR_ERRTEST_ARGNULL(num_cvars, "num_cvars", mpi_errno);
-            MPIR_ERRTEST_ARGNULL(num_pvars, "num_pvars", mpi_errno);
-            MPIR_ERRTEST_ARGNULL(num_categories, "num_categories", mpi_errno);
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-        }
-        MPID_END_ERROR_CHECKS
+    /* ... body of routine ...  */
+    if (cat_index < 0 || cat_index >= utarray_len(cat_table)) {
+        mpi_errno = MPI_T_ERR_INVALID_INDEX;
+        goto fn_fail;
     }
-#   endif /* HAVE_ERROR_CHECKING */
 
-    /* ... body of routine ...  */
+    cat_table_entry_t *cat;
+    cat = (cat_table_entry_t *)utarray_eltptr(cat_table, cat_index);
+    MPIR_T_strncpy(name, cat->name, name_len);
+    MPIR_T_strncpy(desc, cat->desc, desc_len);
+
+    if (num_cvars != NULL)
+        *num_cvars = utarray_len(cat->cvar_indices);
 
-    mpi_errno = MPIR_T_category_get_info_impl(cat_index, name, name_len, desc, desc_len, num_cvars, num_pvars, num_categories);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    if (num_pvars != NULL)
+        *num_pvars = utarray_len(cat->pvar_indices);
 
+    if (num_categories != NULL)
+        *num_categories = utarray_len(cat->subcat_indices);
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_CATEGORY_GET_INFO);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
@@ -126,7 +105,8 @@ fn_fail:
     {
         mpi_errno = MPIR_Err_create_code(
             mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER,
-            "**mpi_t_category_get_info", "**mpi_t_category_get_info %d %p %p %p %p %p %p %p", cat_index, name, name_len, desc, desc_len, num_cvars, num_pvars, num_categories);
+            "**mpi_t_category_get_info", "**mpi_t_category_get_info %d %p %p %p %p %p %p %p",
+            cat_index, name, name_len, desc, desc_len, num_cvars, num_pvars, num_categories);
     }
 #   endif
     mpi_errno = MPIR_Err_return_comm(NULL, FCNAME, mpi_errno);
diff --git a/src/mpi_t/cat_get_num.c b/src/mpi_t/cat_get_num.c
index 6e796db..f0f3e9c 100644
--- a/src/mpi_t/cat_get_num.c
+++ b/src/mpi_t/cat_get_num.c
@@ -21,26 +21,6 @@
 #ifndef MPICH_MPI_FROM_PMPI
 #undef MPI_T_category_get_num
 #define MPI_T_category_get_num PMPI_T_category_get_num
-
-/* any non-MPI functions go here, especially non-static ones */
-
-#undef FUNCNAME
-#define FUNCNAME MPIR_T_category_get_num_impl
-#undef FCNAME
-#define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPIR_T_category_get_num_impl(int *num_cat)
-{
-    int mpi_errno = MPI_SUCCESS;
-
-    /* we don't support any categories right now */
-    *num_cat = 0;
-
-fn_exit:
-    return mpi_errno;
-fn_fail:
-    goto fn_exit;
-}
-
 #endif /* MPICH_MPI_FROM_PMPI */
 
 #undef FUNCNAME
@@ -62,48 +42,22 @@ Output Parameters:
 int MPI_T_category_get_num(int *num_cat)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CATEGORY_GET_NUM);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CATEGORY_GET_NUM);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_CATEGORY_GET_NUM);
 
-    /* Validate parameters, especially handles needing to be converted */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            MPIR_ERRTEST_ARGNULL(num_cat, "num_cat", mpi_errno);
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
     /* ... body of routine ...  */
 
-    mpi_errno = MPIR_T_category_get_num_impl(num_cat);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    if (*num_cat)
+        *num_cat = utarray_len(cat_table);
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_CATEGORY_GET_NUM);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/cat_get_pv.c b/src/mpi_t/cat_get_pv.c
index aeaf630..9190ec5 100644
--- a/src/mpi_t/cat_get_pv.c
+++ b/src/mpi_t/cat_get_pv.c
@@ -30,9 +30,17 @@
 #define FCNAME MPIU_QUOTE(FUNCNAME)
 int MPIR_T_category_get_pvars_impl(int cat_index, int len, int indices[])
 {
-    int mpi_errno = MPI_ERR_INTERN;
+    int mpi_errno = MPI_SUCCESS;
+    cat_table_entry_t *cat;
+    int i, num_pvars, count;
+
+    cat = (cat_table_entry_t *)utarray_eltptr(cat_table, cat_index);
+    num_pvars = utarray_len(cat->pvar_indices);
+    count = len < num_pvars ? len : num_pvars;
 
-    /* TODO implement this function */
+    for (i = 0; i < count; i++) {
+        indices[i] = *(int *)utarray_eltptr(cat->pvar_indices, i);
+    }
 
 fn_exit:
     return mpi_errno;
@@ -65,39 +73,31 @@ Output Parameters:
 int MPI_T_category_get_pvars(int cat_index, int len, int indices[])
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CATEGORY_GET_PVARS);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CATEGORY_GET_PVARS);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_CATEGORY_GET_PVARS);
 
-    /* Validate parameters, especially handles needing to be converted */
+    /* Validate parameters */
 #   ifdef HAVE_ERROR_CHECKING
     {
         MPID_BEGIN_ERROR_CHECKS
         {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            if (len != 0)
+                MPIR_ERRTEST_ARGNULL(indices, "indices", mpi_errno);
         }
         MPID_END_ERROR_CHECKS
     }
 #   endif /* HAVE_ERROR_CHECKING */
 
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
+    /* ... body of routine ...  */
+    if (cat_index < 0 || cat_index >= utarray_len(cat_table)) {
+        mpi_errno = MPI_T_ERR_INVALID_INDEX;
+        goto fn_fail;
     }
-#   endif /* HAVE_ERROR_CHECKING */
 
-    /* ... body of routine ...  */
+    if (len == 0) goto fn_exit;
 
     mpi_errno = MPIR_T_category_get_pvars_impl(cat_index, len, indices);
     if (mpi_errno) MPIU_ERR_POP(mpi_errno);
@@ -106,7 +106,7 @@ int MPI_T_category_get_pvars(int cat_index, int len, int indices[])
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_CATEGORY_GET_PVARS);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/cvar_get_info.c b/src/mpi_t/cvar_get_info.c
index e105fc2..6d17e6d 100644
--- a/src/mpi_t/cvar_get_info.c
+++ b/src/mpi_t/cvar_get_info.c
@@ -21,64 +21,6 @@
 #ifndef MPICH_MPI_FROM_PMPI
 #undef MPI_T_cvar_get_info
 #define MPI_T_cvar_get_info PMPI_T_cvar_get_info
-
-/* any non-MPI functions go here, especially non-static ones */
-
-#undef FUNCNAME
-#define FUNCNAME MPIR_T_cvar_get_info_impl
-#undef FCNAME
-#define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPIR_T_cvar_get_info_impl(int cvar_index, char *name, int *name_len, int *verbosity, MPI_Datatype *datatype, MPI_T_enum *enumtype, char *desc, int *desc_len, int *binding, int *scope)
-{
-    int mpi_errno = MPI_SUCCESS;
-    struct MPIR_Param_t *p = NULL;
-
-    /* TODO convert to real error handling */
-    MPIU_Assert(cvar_index >= 0);
-    MPIU_Assert(cvar_index < MPIR_PARAM_NUM_PARAMS);
-
-    p = &MPIR_Param_params[cvar_index];
-    MPIU_Tool_strncpy(name, p->name, name_len);
-    MPIU_Tool_strncpy(desc, p->description, desc_len);
-
-    /* we do not currently have verbosity levels, so the draft standard says to
-     * return USER_BASIC */
-    *verbosity = MPI_T_VERBOSITY_USER_BASIC;
-
-    switch (p->default_val.type) {
-        case MPIR_PARAM_TYPE_INT:
-            *datatype = MPI_INT;
-            break;
-        case MPIR_PARAM_TYPE_DOUBLE:
-            *datatype = MPI_DOUBLE;
-            break;
-        case MPIR_PARAM_TYPE_BOOLEAN:
-            *datatype = MPI_INT;
-            break;
-        case MPIR_PARAM_TYPE_STRING:
-            *datatype = MPI_CHAR;
-            break;
-        case MPIR_PARAM_TYPE_RANGE:
-            /* type is int, but count should be 2 */
-            *datatype = MPI_INT;
-            break;
-        default:
-            /* FIXME the error handling code may not have been setup yet */
-            MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_INTERN, "**intern", "**intern %s", "unexpected parameter type");
-            break;
-    }
-
-    *binding = MPI_T_BIND_NO_OBJECT;
-
-    /* FIXME a bit of a fib, may not actually be true for any given cvar */
-    *scope = MPI_T_SCOPE_LOCAL;
-
-fn_exit:
-    return mpi_errno;
-fn_fail:
-    goto fn_exit;
-}
-
 #endif /* MPICH_MPI_FROM_PMPI */
 
 #undef FUNCNAME
@@ -110,53 +52,61 @@ Output Parameters:
 
 .N Errors
 @*/
-int MPI_T_cvar_get_info(int cvar_index, char *name, int *name_len, int *verbosity, MPI_Datatype *datatype, MPI_T_enum *enumtype, char *desc, int *desc_len, int *binding, int *scope)
+int MPI_T_cvar_get_info(int cvar_index, char *name, int *name_len,
+    int *verbosity, MPI_Datatype *datatype, MPI_T_enum *enumtype,
+    char *desc, int *desc_len, int *bind, int *scope)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CVAR_GET_INFO);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CVAR_GET_INFO);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_CVAR_GET_INFO);
 
-    /* Validate parameters, especially handles needing to be converted */
+    /* Validate parameters */
 #   ifdef HAVE_ERROR_CHECKING
     {
         MPID_BEGIN_ERROR_CHECKS
         {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            /* Do not do *_TEST_ARGNULL for pointer arguments, since this is
+             * allowed or will be allowed by MPI_T standard.
+             */
         }
         MPID_END_ERROR_CHECKS
     }
 #   endif /* HAVE_ERROR_CHECKING */
 
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            MPIR_ERRTEST_ARGNULL(verbosity, "verbosity", mpi_errno);
-            MPIR_ERRTEST_ARGNULL(binding, "binding", mpi_errno);
-            MPIR_ERRTEST_ARGNULL(scope, "scope", mpi_errno);
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-        }
-        MPID_END_ERROR_CHECKS
+    /* ... body of routine ...  */
+    if (cvar_index < 0 || cvar_index >= utarray_len(cvar_table)) {
+        mpi_errno = MPI_T_ERR_INVALID_INDEX;
+        goto fn_fail;
     }
-#   endif /* HAVE_ERROR_CHECKING */
 
-    /* ... body of routine ...  */
+    const cvar_table_entry_t *cvar;
+    cvar = (cvar_table_entry_t *)utarray_eltptr(cvar_table, cvar_index);
+
+    MPIR_T_strncpy(name, cvar->name,  name_len);
+    MPIR_T_strncpy(desc, cvar->desc,  desc_len);
 
-    mpi_errno = MPIR_T_cvar_get_info_impl(cvar_index, name, name_len, verbosity, datatype, enumtype, desc, desc_len, binding, scope);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    if (verbosity != NULL)
+        *verbosity = cvar->verbosity;
 
+    if (datatype != NULL)
+        *datatype = cvar->datatype;
+
+    if (enumtype != NULL)
+        *enumtype = cvar->enumtype;
+
+    if (bind != NULL)
+        *bind = cvar->bind;
+
+    if (scope != NULL)
+        *scope = cvar->scope;
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_CVAR_GET_INFO);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
@@ -165,9 +115,10 @@ fn_fail:
     {
         mpi_errno = MPIR_Err_create_code(
             mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER,
-            "**mpi_t_cvar_get_info", "**mpi_t_cvar_get_info %d %p %p %p %p %p %p %p %p %p", cvar_index, name, name_len, verbosity, datatype, enumtype, desc, desc_len, binding, scope);
+            "**mpi_t_cvar_get_info", "**mpi_t_cvar_get_info %d %p %p %p %p %p %p %p %p %p",
+            cvar_index, name, name_len, verbosity, datatype, enumtype, desc, desc_len, bind, scope);
     }
-#   endif
+#endif
     mpi_errno = MPIR_Err_return_comm(NULL, FCNAME, mpi_errno);
     goto fn_exit;
     /* --END ERROR HANDLING-- */
diff --git a/src/mpi_t/cvar_get_num.c b/src/mpi_t/cvar_get_num.c
index 981b1f2..c1291b6 100644
--- a/src/mpi_t/cvar_get_num.c
+++ b/src/mpi_t/cvar_get_num.c
@@ -21,25 +21,6 @@
 #ifndef MPICH_MPI_FROM_PMPI
 #undef MPI_T_cvar_get_num
 #define MPI_T_cvar_get_num PMPI_T_cvar_get_num
-
-/* any non-MPI functions go here, especially non-static ones */
-
-#undef FUNCNAME
-#define FUNCNAME MPIR_T_cvar_get_num_impl
-#undef FCNAME
-#define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPIR_T_cvar_get_num_impl(int *num_cvar)
-{
-    int mpi_errno = MPI_SUCCESS;
-
-    *num_cvar = MPIR_PARAM_NUM_PARAMS;
-
-fn_exit:
-    return mpi_errno;
-fn_fail:
-    goto fn_exit;
-}
-
 #endif /* MPICH_MPI_FROM_PMPI */
 
 #undef FUNCNAME
@@ -61,48 +42,22 @@ Output Parameters:
 int MPI_T_cvar_get_num(int *num_cvar)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CVAR_GET_NUM);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CVAR_GET_NUM);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_CVAR_GET_NUM);
 
-    /* Validate parameters, especially handles needing to be converted */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            MPIR_ERRTEST_ARGNULL(num_cvar, "num_cvar", mpi_errno);
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
     /* ... body of routine ...  */
 
-    mpi_errno = MPIR_T_cvar_get_num_impl(num_cvar);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    if (num_cvar != NULL)
+        *num_cvar = utarray_len(cvar_table);
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_CVAR_GET_NUM);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/cvar_handle_alloc.c b/src/mpi_t/cvar_handle_alloc.c
index 9fe3989..737815c 100644
--- a/src/mpi_t/cvar_handle_alloc.c
+++ b/src/mpi_t/cvar_handle_alloc.c
@@ -31,27 +31,33 @@
 int MPIR_T_cvar_handle_alloc_impl(int cvar_index, void *obj_handle, MPI_T_cvar_handle *handle, int *count)
 {
     int mpi_errno = MPI_SUCCESS;
+    MPIR_T_cvar_handle_t *hnd;
+
     MPIU_CHKPMEM_DECL(1);
 
-    *handle = MPI_T_CVAR_HANDLE_NULL;
+    cvar_table_entry_t *cvar = (cvar_table_entry_t *) utarray_eltptr(cvar_table, cvar_index);
 
-    MPIU_Assert(cvar_index >= 0);
-    MPIU_Assert(cvar_index < MPIR_PARAM_NUM_PARAMS);
+    /* Allocate handle memory */
+    MPIU_CHKPMEM_MALLOC(hnd, MPIR_T_cvar_handle_t*, sizeof(*hnd), mpi_errno, "control variable handle");
 
-    *handle = NULL;
-    MPIU_CHKPMEM_MALLOC(*handle, MPI_T_cvar_handle, sizeof(**handle), mpi_errno, "control var handle");
-    (*handle)->p = &MPIR_Param_params[cvar_index];
+    /* It is time to fix addr and count if they are unknown */
+    if (cvar->get_count != NULL)
+        cvar->get_count(obj_handle, count);
+    else
+        *count = cvar->count;
 
-    if ((*handle)->p->default_val.type == MPIR_PARAM_TYPE_RANGE) {
-        *count = 2;
-    }
-    else if ((*handle)->p->default_val.type == MPIR_PARAM_TYPE_STRING) {
-        /* TODO it might be useful to be able to override on a per-var basis */
-        *count = MPIR_PARAM_MAX_STRLEN;
-    }
-    else {
-        *count = 1;
-    }
+    hnd->count = *count;
+
+    if (cvar->get_addr != NULL)
+        cvar->get_addr(obj_handle, &(hnd->addr));
+    else
+        hnd->addr = cvar->addr;
+
+    /* Cache other fields */
+    hnd->datatype = cvar->datatype;
+    hnd->scope = cvar->scope;
+
+    *handle = hnd;
 
     MPIU_CHKPMEM_COMMIT();
 fn_exit:
@@ -87,39 +93,29 @@ Output Parameters:
 int MPI_T_cvar_handle_alloc(int cvar_index, void *obj_handle, MPI_T_cvar_handle *handle, int *count)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CVAR_HANDLE_ALLOC);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CVAR_HANDLE_ALLOC);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_CVAR_HANDLE_ALLOC);
 
-    /* Validate parameters, especially handles needing to be converted */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
+    /* Validate parameters */
 #   ifdef HAVE_ERROR_CHECKING
     {
         MPID_BEGIN_ERROR_CHECKS
         {
+            MPIR_ERRTEST_ARGNULL(handle, "handle", mpi_errno);
             MPIR_ERRTEST_ARGNULL(count, "count", mpi_errno);
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
         }
         MPID_END_ERROR_CHECKS
     }
 #   endif /* HAVE_ERROR_CHECKING */
 
     /* ... body of routine ...  */
+    if (cvar_index < 0 || cvar_index >= utarray_len(cvar_table)) {
+        mpi_errno = MPI_T_ERR_INVALID_INDEX;
+        goto fn_fail;
+    }
 
     mpi_errno = MPIR_T_cvar_handle_alloc_impl(cvar_index, obj_handle, handle, count);
     if (mpi_errno) MPIU_ERR_POP(mpi_errno);
@@ -128,7 +124,7 @@ int MPI_T_cvar_handle_alloc(int cvar_index, void *obj_handle, MPI_T_cvar_handle
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_CVAR_HANDLE_ALLOC);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/cvar_handle_free.c b/src/mpi_t/cvar_handle_free.c
index 99e1495..81c5f11 100644
--- a/src/mpi_t/cvar_handle_free.c
+++ b/src/mpi_t/cvar_handle_free.c
@@ -21,26 +21,6 @@
 #ifndef MPICH_MPI_FROM_PMPI
 #undef MPI_T_cvar_handle_free
 #define MPI_T_cvar_handle_free PMPI_T_cvar_handle_free
-
-/* any non-MPI functions go here, especially non-static ones */
-
-#undef FUNCNAME
-#define FUNCNAME MPIR_T_cvar_handle_free_impl
-#undef FCNAME
-#define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPIR_T_cvar_handle_free_impl(MPI_T_cvar_handle *handle)
-{
-    int mpi_errno = MPI_SUCCESS;
-
-    MPIU_Free(*handle);
-    *handle = MPI_T_CVAR_HANDLE_NULL;
-
-fn_exit:
-    return mpi_errno;
-fn_fail:
-    goto fn_exit;
-}
-
 #endif /* MPICH_MPI_FROM_PMPI */
 
 #undef FUNCNAME
@@ -62,48 +42,25 @@ Input/Output Parameters:
 int MPI_T_cvar_handle_free(MPI_T_cvar_handle *handle)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CVAR_HANDLE_FREE);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CVAR_HANDLE_FREE);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_CVAR_HANDLE_FREE);
 
-    /* Validate parameters, especially handles needing to be converted */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
     /* ... body of routine ...  */
 
-    mpi_errno = MPIR_T_cvar_handle_free_impl(handle);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    if (handle != NULL) {
+        MPIR_T_cvar_handle_t *hnd = *handle;
+        MPIU_Free(hnd);
+        *handle = MPI_T_CVAR_HANDLE_NULL;
+    }
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_CVAR_HANDLE_FREE);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/cvar_read.c b/src/mpi_t/cvar_read.c
index 8f8ca95..7c92bb2 100644
--- a/src/mpi_t/cvar_read.c
+++ b/src/mpi_t/cvar_read.c
@@ -31,43 +31,42 @@
 int MPIR_T_cvar_read_impl(MPI_T_cvar_handle handle, void *buf)
 {
     int mpi_errno = MPI_SUCCESS;
-    struct MPIR_Param_t *p = handle->p;
-
-    switch (p->default_val.type) {
-        case MPIR_PARAM_TYPE_INT:
-            {
-                int *i_buf = buf;
-                *i_buf = *(int *)p->val_p;
-            }
-            break;
-        case MPIR_PARAM_TYPE_DOUBLE:
-            {
-                double *d_buf = buf;
-                *d_buf = *(double *)p->val_p;
-            }
-            break;
-        case MPIR_PARAM_TYPE_BOOLEAN:
-            {
-                int *i_buf = buf;
-                *i_buf = *(int *)p->val_p;
-            }
-            break;
-        case MPIR_PARAM_TYPE_STRING:
-            if (*(char **)p->val_p == NULL) {
-                char *c_buf = buf;
-                c_buf[0] = '\0';
-            }
-            else {
-                MPIU_Strncpy(buf, *(char **)p->val_p, MPIR_PARAM_MAX_STRLEN);
-            }
-            break;
-        case MPIR_PARAM_TYPE_RANGE:
-            MPIU_Memcpy(buf, p->val_p, 2*sizeof(int));
-            break;
-        default:
-            /* FIXME the error handling code may not have been setup yet */
-            MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_INTERN, "**intern", "**intern %s", "unexpected parameter type");
-            break;
+    int i, count;
+    void *addr;
+    MPIR_T_cvar_handle_t *hnd = handle;
+
+    count = hnd->count;
+    addr = hnd->addr;
+    MPIU_Assert(addr != NULL);
+
+    switch (hnd->datatype) {
+    case MPI_INT:
+        for (i = 0; i < count; i++)
+            ((int *)buf)[i] = ((int *)addr)[i];
+        break;
+    case MPI_UNSIGNED:
+        for (i = 0; i < count; i++)
+            ((unsigned *)buf)[i] = ((unsigned *)addr)[i];
+        break;
+    case MPI_UNSIGNED_LONG:
+        for (i = 0; i < count; i++)
+            ((unsigned long *)buf)[i] = ((unsigned long *)addr)[i];
+        break;
+    case MPI_UNSIGNED_LONG_LONG:
+        for (i = 0; i < count; i++)
+            ((unsigned long long *)buf)[i] = ((unsigned long long *)addr)[i];
+        break;
+    case MPI_DOUBLE:
+        for (i = 0; i < count; i++)
+            ((double *)buf)[i] = ((double *)addr)[i];
+        break;
+    case MPI_CHAR:
+        MPIU_Strncpy(buf, addr, count);
+        break;
+    default:
+         /* FIXME the error handling code may not have been setup yet */
+        MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_INTERN, "**intern", "**intern %s", "unexpected parameter type");
+        break;
     }
 
 fn_exit:
@@ -100,33 +99,19 @@ Output Parameters:
 int MPI_T_cvar_read(MPI_T_cvar_handle handle, void *buf)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CVAR_READ);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CVAR_READ);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_CVAR_READ);
 
-    /* Validate parameters, especially handles needing to be converted */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
+    /* Validate parameters */
 #   ifdef HAVE_ERROR_CHECKING
     {
         MPID_BEGIN_ERROR_CHECKS
         {
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            MPIR_ERRTEST_ARGNULL(handle, "handle", mpi_errno);
+            MPIR_ERRTEST_ARGNULL(buf, "buf", mpi_errno);
         }
         MPID_END_ERROR_CHECKS
     }
@@ -141,7 +126,7 @@ int MPI_T_cvar_read(MPI_T_cvar_handle handle, void *buf)
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_CVAR_READ);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/cvar_write.c b/src/mpi_t/cvar_write.c
index 09c0cf3..fb616eb 100644
--- a/src/mpi_t/cvar_write.c
+++ b/src/mpi_t/cvar_write.c
@@ -31,48 +31,51 @@
 int MPIR_T_cvar_write_impl(MPI_T_cvar_handle handle, void *buf)
 {
     int mpi_errno = MPI_SUCCESS;
-    struct MPIR_Param_t *p = handle->p;
-
-    switch (p->default_val.type) {
-        case MPIR_PARAM_TYPE_INT:
-            {
-                int *ip_buf = (int *)p->val_p;
-                *ip_buf = *(int *)buf;
-            }
-            break;
-        case MPIR_PARAM_TYPE_DOUBLE:
-            {
-                double *dp_buf = (double *)p->val_p;
-                *dp_buf = *(double *)buf;
-            }
-            break;
-        case MPIR_PARAM_TYPE_BOOLEAN:
-            {
-                int *ip_buf = (int *)p->val_p;
-                *ip_buf = *(int *)buf;
-            }
-            break;
-        case MPIR_PARAM_TYPE_STRING:
-            {
-                char **lval_ptr = (char **)p->val_p;
-                if (*lval_ptr != NULL) {
-                    /* FIXME these strings are not all (in fact, almost never)
-                     * allocated with malloc/strdup right now */
-                    MPIU_Free(*lval_ptr);
-                    *lval_ptr = NULL;
-                }
-                *lval_ptr = MPIU_Strdup(buf);
-            }
-            break;
-        case MPIR_PARAM_TYPE_RANGE:
-            MPIU_Memcpy(p->val_p, buf, 2*sizeof(int));
-            break;
-        default:
-            /* FIXME the error handling code may not have been setup yet */
-            MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_INTERN, "**intern", "**intern %s", "unexpected parameter type");
-            break;
+    int i, count;
+    void *addr;
+    MPIR_T_cvar_handle_t *hnd = handle;
+
+    if (hnd->scope == MPI_T_SCOPE_CONSTANT) {
+        mpi_errno = MPI_T_ERR_CVAR_SET_NEVER;
+        goto fn_fail;
+    } else if (hnd->scope == MPI_T_SCOPE_READONLY) {
+        mpi_errno = MPI_T_ERR_CVAR_SET_NOT_NOW;
+        goto fn_fail;
     }
 
+    count = hnd->count;
+    addr = hnd->addr;
+    MPIU_Assert(addr != NULL);
+
+    switch (hnd->datatype) {
+    case MPI_INT:
+        for (i = 0; i < count; i++)
+            ((int *)addr)[i] = ((int *)buf)[i];
+        break;
+    case MPI_UNSIGNED:
+        for (i = 0; i < count; i++)
+            ((unsigned *)addr)[i] = ((unsigned *)buf)[i];
+        break;
+    case MPI_UNSIGNED_LONG:
+        for (i = 0; i < count; i++)
+            ((unsigned long *)addr)[i] = ((unsigned long *)buf)[i];
+        break;
+    case MPI_UNSIGNED_LONG_LONG:
+        for (i = 0; i < count; i++)
+            ((unsigned long long *)addr)[i] = ((unsigned long long *)buf)[i];
+        break;
+    case MPI_DOUBLE:
+        for (i = 0; i < count; i++)
+            ((double *)addr)[i] = ((double *)buf)[i];
+        break;
+    case MPI_CHAR:
+        MPIU_Strncpy(addr, buf, count);
+        break;
+    default:
+         /* FIXME the error handling code may not have been setup yet */
+        MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_INTERN, "**intern", "**intern %s", "unexpected parameter type");
+        break;
+    }
 
 fn_exit:
     return mpi_errno;
@@ -102,33 +105,19 @@ Input Parameters:
 int MPI_T_cvar_write(MPI_T_cvar_handle handle, void *buf)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CVAR_WRITE);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_CVAR_WRITE);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_CVAR_WRITE);
 
-    /* Validate parameters, especially handles needing to be converted */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
+   /* Validate parameters */
 #   ifdef HAVE_ERROR_CHECKING
     {
         MPID_BEGIN_ERROR_CHECKS
         {
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            MPIR_ERRTEST_ARGNULL(handle, "handle", mpi_errno);
+            MPIR_ERRTEST_ARGNULL(buf, "buf", mpi_errno);
         }
         MPID_END_ERROR_CHECKS
     }
@@ -143,7 +132,7 @@ int MPI_T_cvar_write(MPI_T_cvar_handle handle, void *buf)
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_CVAR_WRITE);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/enum_get_info.c b/src/mpi_t/enum_get_info.c
index 7289b7f..2e3bd66 100644
--- a/src/mpi_t/enum_get_info.c
+++ b/src/mpi_t/enum_get_info.c
@@ -21,25 +21,6 @@
 #ifndef MPICH_MPI_FROM_PMPI
 #undef MPI_T_enum_get_info
 #define MPI_T_enum_get_info PMPI_T_enum_get_info
-
-/* any non-MPI functions go here, especially non-static ones */
-
-#undef FUNCNAME
-#define FUNCNAME MPIR_T_enum_get_info_impl
-#undef FCNAME
-#define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPIR_T_enum_get_info_impl(MPI_T_enum enumtype, int *num, char *name, int *name_len)
-{
-    int mpi_errno = MPI_ERR_INTERN;
-
-    /* TODO implement this function */
-
-fn_exit:
-    return mpi_errno;
-fn_fail:
-    goto fn_exit;
-}
-
 #endif /* MPICH_MPI_FROM_PMPI */
 
 #undef FUNCNAME
@@ -68,34 +49,22 @@ Output Parameters:
 int MPI_T_enum_get_info(MPI_T_enum enumtype, int *num, char *name, int *name_len)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_ENUM_GET_INFO);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_ENUM_GET_INFO);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_ENUM_GET_INFO);
 
-    /* Validate parameters, especially handles needing to be converted */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
+    /* Validate parameters */
 #   ifdef HAVE_ERROR_CHECKING
     {
         MPID_BEGIN_ERROR_CHECKS
         {
+            MPIR_ERRTEST_ARGNULL(enumtype, "enumtype", mpi_errno);
             MPIR_ERRTEST_ARGNULL(num, "num", mpi_errno);
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            /* Do not do _TEST_ARGNULL for name or name_len, since this is
+             * permitted per MPI_T standard.
+             */
         }
         MPID_END_ERROR_CHECKS
     }
@@ -103,14 +72,14 @@ int MPI_T_enum_get_info(MPI_T_enum enumtype, int *num, char *name, int *name_len
 
     /* ... body of routine ...  */
 
-    mpi_errno = MPIR_T_enum_get_info_impl(enumtype, num, name, name_len);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    *num = utarray_len(enumtype->items);
+    MPIR_T_strncpy(name, enumtype->name, name_len);
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_ENUM_GET_INFO);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
@@ -119,7 +88,8 @@ fn_fail:
     {
         mpi_errno = MPIR_Err_create_code(
             mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER,
-            "**mpi_t_enum_get_info", "**mpi_t_enum_get_info %p %p %p %p", enumtype, num, name, name_len);
+            "**mpi_t_enum_get_info", "**mpi_t_enum_get_info %p %p %p %p",
+            enumtype, num, name, name_len);
     }
 #   endif
     mpi_errno = MPIR_Err_return_comm(NULL, FCNAME, mpi_errno);
diff --git a/src/mpi_t/enum_get_item.c b/src/mpi_t/enum_get_item.c
index 4d8eec4..da7e115 100644
--- a/src/mpi_t/enum_get_item.c
+++ b/src/mpi_t/enum_get_item.c
@@ -21,25 +21,6 @@
 #ifndef MPICH_MPI_FROM_PMPI
 #undef MPI_T_enum_get_item
 #define MPI_T_enum_get_item PMPI_T_enum_get_item
-
-/* any non-MPI functions go here, especially non-static ones */
-
-#undef FUNCNAME
-#define FUNCNAME MPIR_T_enum_get_item_impl
-#undef FCNAME
-#define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPIR_T_enum_get_item_impl(MPI_T_enum enumtype, int indx, int *value, char *name, int *name_len)
-{
-    int mpi_errno = MPI_ERR_INTERN;
-
-    /* TODO implement this function */
-
-fn_exit:
-    return mpi_errno;
-fn_fail:
-    goto fn_exit;
-}
-
 #endif /* MPICH_MPI_FROM_PMPI */
 
 #undef FUNCNAME
@@ -56,7 +37,7 @@ Input Parameters:
 . enumtype - enumeration to be queried (handle)
 
 Output Parameters:
-+ indx - number of the value to be queried in this enumeration (integer)
++ index - number of the value to be queried in this enumeration (integer)
 . value - variable value (integer)
 - name - buffer to return the string containing the name of the enumeration item (string)
 
@@ -66,37 +47,25 @@ Output Parameters:
 
 .N Errors
 @*/
-int MPI_T_enum_get_item(MPI_T_enum enumtype, int indx, int *value, char *name, int *name_len)
+int MPI_T_enum_get_item(MPI_T_enum enumtype, int index, int *value, char *name, int *name_len)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_ENUM_GET_ITEM);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_ENUM_GET_ITEM);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_ENUM_GET_ITEM);
 
-    /* Validate parameters, especially handles needing to be converted */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
+    /* Validate parameters */
 #   ifdef HAVE_ERROR_CHECKING
     {
         MPID_BEGIN_ERROR_CHECKS
         {
+            MPIR_ERRTEST_ARGNULL(enumtype, "enumtype", mpi_errno);
             MPIR_ERRTEST_ARGNULL(value, "value", mpi_errno);
-            MPIR_ERRTEST_ARGNULL(name_len, "name_len", mpi_errno);
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
+            /* Do not do TEST_ARGNULL for name or name_len, since this is
+             * permitted per MPI_T standard.
+             */
         }
         MPID_END_ERROR_CHECKS
     }
@@ -104,14 +73,20 @@ int MPI_T_enum_get_item(MPI_T_enum enumtype, int indx, int *value, char *name, i
 
     /* ... body of routine ...  */
 
-    mpi_errno = MPIR_T_enum_get_item_impl(enumtype, indx, value, name, name_len);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    if (index < 0 || index > utarray_len(enumtype->items) - 1) {
+        mpi_errno = MPI_T_ERR_INVALID_ITEM;
+        goto fn_fail;
+    }
+
+    enum_item_t *item = (enum_item_t *)utarray_eltptr(enumtype->items, index);
+    *value = item->value;
+    MPIR_T_strncpy(name, item->name, name_len);
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_ENUM_GET_ITEM);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
@@ -120,7 +95,8 @@ fn_fail:
     {
         mpi_errno = MPIR_Err_create_code(
             mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER,
-            "**mpi_t_enum_get_item", "**mpi_t_enum_get_item %p %d %p %p %p", enumtype, indx, value, name, name_len);
+            "**mpi_t_enum_get_item", "**mpi_t_enum_get_item %p %d %p %p %p",
+            enumtype, index, value, name, name_len);
     }
 #   endif
     mpi_errno = MPIR_Err_return_comm(NULL, FCNAME, mpi_errno);
diff --git a/src/mpi_t/mpit_finalize.c b/src/mpi_t/mpit_finalize.c
index 59e0cf3..23c3559 100644
--- a/src/mpi_t/mpit_finalize.c
+++ b/src/mpi_t/mpit_finalize.c
@@ -6,9 +6,6 @@
 
 #include "mpiimpl.h"
 
-/* defined in mpit_init_thread.c */
-extern int MPIR_T_init_balance;
-
 /* -- Begin Profiling Symbol Block for routine MPI_T_finalize */
 #if defined(HAVE_PRAGMA_WEAK)
 #pragma weak MPI_T_finalize = PMPI_T_finalize
@@ -27,27 +24,143 @@ extern int MPIR_T_init_balance;
 
 /* any non-MPI functions go here, especially non-static ones */
 
-#undef FUNCNAME
-#define FUNCNAME MPIR_T_finalize_impl
-#undef FCNAME
-#define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPIR_T_finalize_impl(void)
+static void MPIR_T_enum_env_finalize(void)
 {
-    int mpi_errno = MPI_SUCCESS;
+    int i, j;
+    MPIR_T_enum_t *e;
+    enum_item_t *item;
+
+    if (enum_table) {
+        /* Free all entries */
+        for (i = 0; i < utarray_len(enum_table); i++) {
+            e = (MPIR_T_enum_t *)utarray_eltptr(enum_table, i);
+            MPIU_Free((void *)e->name);
+
+            /* Free items in this enum */
+            for (j = 0; j < utarray_len(e->items); j++) {
+                item = (enum_item_t *)utarray_eltptr(e->items, j);
+                MPIU_Free((void *)item->name);
+            }
+
+            utarray_free(e->items);
+        }
 
-    --MPIR_T_init_balance;
-    MPIU_Assert(MPIR_T_init_balance >= 0);
-    if (MPIR_T_init_balance == 0 && MPIR_Process.initialized != MPICH_WITHIN_MPI) {
-        MPIR_T_finalize_pvars();
+        /* Free enum_table itself */
+        utarray_free(enum_table);
+        enum_table = NULL;
+    }
+}
+
+static void MPIR_T_cat_env_finalize(void)
+{
+    int i;
+    cat_table_entry_t *cat;
+
+    if (cat_table) {
+        /* Free all entries */
+        for (i = 0; i < utarray_len(cat_table); i++) {
+            cat = (cat_table_entry_t *)utarray_eltptr(cat_table, i);
+            MPIU_Free((void *)cat->name);
+            MPIU_Free((void *)cat->desc);
+            utarray_free(cat->cvar_indices);
+            utarray_free(cat->pvar_indices);
+            utarray_free(cat->subcat_indices);
+        }
 
-        mpi_errno = MPIR_Param_finalize();
-        if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+        /* Free cat_table itself */
+        utarray_free(cat_table);
+        cat_table = NULL;
     }
 
-fn_exit:
-    return mpi_errno;
-fn_fail:
-    goto fn_exit;
+    if (cat_hash) {
+        name2index_hash_t *current, *tmp;
+        /* Free all entries */
+        HASH_ITER(hh, cat_hash, current, tmp) {
+            HASH_DEL(cat_hash, current);
+            MPIU_Free(current);
+        }
+
+        /* Free cat_hash itself */
+        HASH_CLEAR(hh, cat_hash);
+        cat_hash = NULL;
+    }
+}
+
+
+static void MPIR_T_cvar_env_finalize(void)
+{
+    int i;
+    cvar_table_entry_t *cvar;
+
+    MPIR_Param_finalize();
+
+    if (cvar_table) {
+        /* Free all entries */
+        for (i = 0; i < utarray_len(cvar_table); i++) {
+            cvar = (cvar_table_entry_t *)utarray_eltptr(cvar_table, i);
+            MPIU_Free((void *)cvar->name);
+            MPIU_Free((void *)cvar->desc);
+        }
+
+        /* Free pvar_table itself */
+        utarray_free(cvar_table);
+        cvar_table = NULL;
+    }
+
+    if (cvar_hash) {
+        name2index_hash_t *current, *tmp;
+        /* Free all entries */
+        HASH_ITER(hh, cvar_hash, current, tmp) {
+            HASH_DEL(cvar_hash, current);
+            MPIU_Free(current);
+        }
+
+        /* Free cvar_hash itself */
+        HASH_CLEAR(hh, cvar_hash);
+        cvar_hash = NULL;
+    }
+}
+
+static void MPIR_T_pvar_env_finalize(void)
+{
+    int i;
+    pvar_table_entry_t *pvar;
+
+    if (pvar_table) {
+        /* Free all entries */
+        for (i = 0; i < utarray_len(pvar_table); i++) {
+            pvar = (pvar_table_entry_t *)utarray_eltptr(pvar_table, i);
+            MPIU_Free((void *)pvar->name);
+            MPIU_Free((void *)pvar->desc);
+        }
+
+        /* Free pvar_table itself */
+        utarray_free(pvar_table);
+        pvar_table = NULL;
+    }
+
+    for (i = 0; i < MPIR_T_PVAR_CLASS_NUMBER; i++) {
+        if (pvar_hashs[i]) {
+            name2index_hash_t *current, *tmp;
+            /* Free all entries */
+            HASH_ITER(hh, pvar_hashs[i], current, tmp) {
+                HASH_DEL(pvar_hashs[i], current);
+                MPIU_Free(current);
+            }
+
+            /* Free pvar_hashs[i] itself */
+            HASH_CLEAR(hh, pvar_hashs[i]);
+            pvar_hashs[i] = NULL;
+        }
+    }
+}
+
+void MPIR_T_env_finalize(void)
+{
+    MPIR_T_enum_env_finalize();
+    MPIR_T_cvar_env_finalize();
+    MPIR_T_pvar_env_finalize();
+    MPIR_T_cat_env_finalize();
 }
 
 #endif /* MPICH_MPI_FROM_PMPI */
@@ -68,48 +181,27 @@ MPI_T_finalize - XXX description here
 int MPI_T_finalize(void)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_FINALIZE);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_FINALIZE);
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_FINALIZE);
 
-    /* Validate parameters, especially handles needing to be converted */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
+    /* ... body of routine ...  */
 
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
+    --MPIR_T_init_balance;
+    if (MPIR_T_init_balance < 0) {
+        mpi_errno = MPI_T_ERR_NOT_INITIALIZED;
+        goto fn_fail;
     }
-#   endif /* HAVE_ERROR_CHECKING */
 
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
+    if (MPIR_T_init_balance == 0) {
+        MPIR_T_THREAD_CS_FINALIZE();
+        MPIR_T_env_finalize();
     }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* ... body of routine ...  */
-
-    mpi_errno = MPIR_T_finalize_impl();
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_FINALIZE);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/mpit_init_thread.c b/src/mpi_t/mpit_init_thread.c
index 037c09a..31f0ec5 100644
--- a/src/mpi_t/mpit_init_thread.c
+++ b/src/mpi_t/mpit_init_thread.c
@@ -23,43 +23,57 @@
 #define MPI_T_init_thread PMPI_T_init_thread
 
 /* any non-MPI functions go here, especially non-static ones */
+static inline void MPIR_T_enum_env_init(void)
+{
+    static const UT_icd enum_table_entry_icd =
+        {sizeof(MPIR_T_enum_t), NULL, NULL, NULL};
 
-/* a counter that keeps track of the relative balance of calls to
- * MPI_T_init_thread and MPI_T_finalize */
-int MPIR_T_init_balance = 0;
+    utarray_new(enum_table, &enum_table_entry_icd);
+}
 
-/* returns true iff the MPI_T_ interface is currently initialized */
-int MPIR_T_is_initialized(void)
+static inline void MPIR_T_cat_env_init(void)
 {
-    return (MPIR_T_init_balance > 0);
+    static const UT_icd cat_table_entry_icd =
+                    {sizeof(cat_table_entry_t), NULL, NULL, NULL};
+
+    utarray_new(cat_table, &cat_table_entry_icd);
+    cat_hash = NULL;
+    cat_stamp = 0;
 }
 
-#undef FUNCNAME
-#define FUNCNAME MPIR_T_init_thread_impl
-#undef FCNAME
-#define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPIR_T_init_thread_impl(int required, int *provided)
+static inline void MPIR_T_cvar_env_init(void)
 {
-    int mpi_errno = MPI_SUCCESS;
+    static const UT_icd cvar_table_entry_icd =
+                    {sizeof(cvar_table_entry_t), NULL, NULL, NULL};
 
-    /* TODO deal with required/provided and threading initialization */
-    /* always claim SINGLE for now, since it's a lot of work to separately
-     * initialize all of the threading code without initializing the whole MPI
-     * library right now */
-    *provided = MPI_THREAD_SINGLE;
+    utarray_new(cvar_table, &cvar_table_entry_icd);
+    cvar_hash = NULL;
+    MPIR_Param_init_params();
+}
 
-    ++MPIR_T_init_balance;
-    if (MPIR_T_init_balance == 1) {
-        /* _init_params is idempotent, so it's OK if we call it after or before
-         * MPI_Init does */
-        mpi_errno = MPIR_Param_init_params();
-        if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+static inline void MPIR_T_pvar_env_init(void)
+{
+    int i;
+    static const UT_icd pvar_table_entry_icd =
+                    {sizeof(pvar_table_entry_t), NULL, NULL, NULL};
+
+    utarray_new(pvar_table, &pvar_table_entry_icd);
+    for (i = 0; i < MPIR_T_PVAR_CLASS_NUMBER; i++) {
+        pvar_hashs[i] = NULL;
     }
+}
 
-fn_exit:
-    return mpi_errno;
-fn_fail:
-    goto fn_exit;
+void MPIR_T_env_init(void)
+{
+    static int initialized = FALSE;
+
+    if (!initialized) {
+        initialized = TRUE;
+        MPIR_T_enum_env_init();
+        MPIR_T_cat_env_init();
+        MPIR_T_cvar_env_init();
+        MPIR_T_pvar_env_init();
+    }
 }
 
 #endif /* MPICH_MPI_FROM_PMPI */
@@ -86,48 +100,33 @@ Output Parameters:
 int MPI_T_init_thread(int required, int *provided)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_INIT_THREAD);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_INIT_THREAD);
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_INIT_THREAD);
 
-    /* Validate parameters, especially handles needing to be converted */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
+    /* ... body of routine ...  */
 
-    /* Convert MPI object handles to object pointers */
+#ifdef HAVE_RUNTIME_THREADCHECK
+    MPIR_T_is_threaded = (required == MPI_THREAD_MULTIPLE);
+#endif
 
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            MPIR_ERRTEST_ARGNULL(provided, "provided", mpi_errno);
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-        }
-        MPID_END_ERROR_CHECKS
+    if (provided != NULL) {
+	    /* This must be min(required,MPICH_THREAD_LEVEL) if runtime
+	       control of thread level is available */
+	    *provided = (MPICH_THREAD_LEVEL < required) ?
+	        MPICH_THREAD_LEVEL : required;
     }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* ... body of routine ...  */
 
-    mpi_errno = MPIR_T_init_thread_impl(required, provided);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    ++MPIR_T_init_balance;
+    if (MPIR_T_init_balance == 1) {
+        MPIR_T_THREAD_CS_INIT();
+        MPIR_T_env_init();
+    }
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_INIT_THREAD);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/pvar_get_info.c b/src/mpi_t/pvar_get_info.c
index f8daf41..67bd80f 100644
--- a/src/mpi_t/pvar_get_info.c
+++ b/src/mpi_t/pvar_get_info.c
@@ -21,41 +21,6 @@
 #ifndef MPICH_MPI_FROM_PMPI
 #undef MPI_T_pvar_get_info
 #define MPI_T_pvar_get_info PMPI_T_pvar_get_info
-
-/* any non-MPI functions go here, especially non-static ones */
-
-#undef FUNCNAME
-#define FUNCNAME MPIR_T_pvar_get_info_impl
-#undef FCNAME
-#define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPIR_T_pvar_get_info_impl(int pvar_index, char *name, int *name_len, int *verbosity, int *var_class, MPI_Datatype *datatype, MPI_T_enum *enumtype, char *desc, int *desc_len, int *binding, int *readonly, int *continuous, int *atomic)
-{
-    int mpi_errno = MPI_SUCCESS;
-    struct MPIR_T_pvar_info *info = NULL;
-
-    MPIU_Assert(pvar_index >= 0);
-
-    mpi_errno = MPIR_T_get_pvar_info_by_idx(pvar_index, &info);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
-    MPIU_Tool_strncpy(name, info->name, name_len);
-    MPIU_Tool_strncpy(desc, info->desc, desc_len);
-
-    *verbosity  = info->verbosity;
-    *var_class  = info->varclass;
-    *datatype   = info->dtype;
-    *enumtype   = info->etype;
-    *binding    = info->binding;
-    *readonly   = info->readonly;
-    *continuous = info->continuous;
-    *atomic     = info->atomic;
-
-fn_exit:
-    return mpi_errno;
-fn_fail:
-    goto fn_exit;
-}
-
 #endif /* MPICH_MPI_FROM_PMPI */
 
 #undef FUNCNAME
@@ -90,56 +55,79 @@ Output Parameters:
 
 .N Errors
 @*/
-int MPI_T_pvar_get_info(int pvar_index, char *name, int *name_len, int *verbosity, int *var_class, MPI_Datatype *datatype, MPI_T_enum *enumtype, char *desc, int *desc_len, int *binding, int *readonly, int *continuous, int *atomic)
+int MPI_T_pvar_get_info(int pvar_index, char *name, int *name_len, int *verbosity,
+    int *var_class, MPI_Datatype *datatype, MPI_T_enum *enumtype, char *desc,
+    int *desc_len, int *bind, int *readonly, int *continuous, int *atomic)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_GET_INFO);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_GET_INFO);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_PVAR_GET_INFO);
 
-    /* Validate parameters, especially handles needing to be converted */
+    /* Validate parameters */
 #   ifdef HAVE_ERROR_CHECKING
     {
         MPID_BEGIN_ERROR_CHECKS
         {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            /* Do not do _TEST_ARGNULL for any argument, since this is
+             * allowed or will be allowed by MPI_T standard.
+             */
         }
         MPID_END_ERROR_CHECKS
     }
 #   endif /* HAVE_ERROR_CHECKING */
 
-    /* Convert MPI object handles to object pointers */
+    /* ... body of routine ...  */
+    const pvar_table_entry_t *entry;
 
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            MPIR_ERRTEST_ARGNULL(verbosity, "verbosity", mpi_errno);
-            MPIR_ERRTEST_ARGNULL(var_class, "var_class", mpi_errno);
-            MPIR_ERRTEST_ARGNULL(binding, "binding", mpi_errno);
-            MPIR_ERRTEST_ARGNULL(readonly, "readonly", mpi_errno);
-            MPIR_ERRTEST_ARGNULL(continuous, "continuous", mpi_errno);
-            MPIR_ERRTEST_ARGNULL(atomic, "atomic", mpi_errno);
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-        }
-        MPID_END_ERROR_CHECKS
+    if (pvar_index < 0 || pvar_index >= utarray_len(pvar_table)) {
+        mpi_errno = MPI_T_ERR_INVALID_INDEX;
+        goto fn_fail;
     }
-#   endif /* HAVE_ERROR_CHECKING */
 
-    /* ... body of routine ...  */
+    entry = (pvar_table_entry_t *) utarray_eltptr(pvar_table, pvar_index);
+    if (!entry->active) {
+        mpi_errno = MPI_T_ERR_INVALID_INDEX;
+        goto fn_fail;
+    }
+
+    const pvar_table_entry_t *info;
+    info = (pvar_table_entry_t *) utarray_eltptr(pvar_table, pvar_index);
+
+    MPIR_T_strncpy(name, info->name, name_len);
+    MPIR_T_strncpy(desc, info->desc, desc_len);
+
+    if (verbosity != NULL)
+        *verbosity = info->verbosity;
+
+    if (var_class != NULL)
+        *var_class = info->varclass;
+
+    if (datatype != NULL)
+        *datatype = info->datatype;
+
+    if (enumtype != NULL)
+        *enumtype = info->enumtype;
+
+    if (bind != NULL)
+        *bind = info->bind;
+
+    if (readonly != NULL)
+        *readonly = info->flags & MPIR_T_PVAR_FLAG_READONLY;
+
+    if (continuous != NULL)
+        *continuous = info->flags & MPIR_T_PVAR_FLAG_CONTINUOUS;
 
-    mpi_errno = MPIR_T_pvar_get_info_impl(pvar_index, name, name_len, verbosity, var_class, datatype, enumtype, desc, desc_len, binding, readonly, continuous, atomic);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    if (atomic != NULL)
+        *atomic = info->flags & MPIR_T_PVAR_FLAG_ATOMIC;
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_PVAR_GET_INFO);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
@@ -148,7 +136,9 @@ fn_fail:
     {
         mpi_errno = MPIR_Err_create_code(
             mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER,
-            "**mpi_t_pvar_get_info", "**mpi_t_pvar_get_info %d %p %p %p %p %p %p %p %p %p %p %p %p", pvar_index, name, name_len, verbosity, var_class, datatype, enumtype, desc, desc_len, binding, readonly, continuous, atomic);
+            "**mpi_t_pvar_get_info", "**mpi_t_pvar_get_info %d %p %p %p %p %p %p %p %p %p %p %p %p",
+            pvar_index, name, name_len, verbosity, var_class, datatype, enumtype, desc, desc_len,
+            bind, readonly, continuous, atomic);
     }
 #   endif
     mpi_errno = MPIR_Err_return_comm(NULL, FCNAME, mpi_errno);
diff --git a/src/mpi_t/pvar_get_num.c b/src/mpi_t/pvar_get_num.c
index 56a4179..c4a8fdd 100644
--- a/src/mpi_t/pvar_get_num.c
+++ b/src/mpi_t/pvar_get_num.c
@@ -21,27 +21,6 @@
 #ifndef MPICH_MPI_FROM_PMPI
 #undef MPI_T_pvar_get_num
 #define MPI_T_pvar_get_num PMPI_T_pvar_get_num
-
-/* any non-MPI functions go here, especially non-static ones */
-
-#undef FUNCNAME
-#define FUNCNAME MPIR_T_pvar_get_num_impl
-#undef FCNAME
-#define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPIR_T_pvar_get_num_impl(int *num_pvar)
-{
-    int mpi_errno = MPI_SUCCESS;
-
-    /* no performance vars implemented at the moment */
-    mpi_errno = MPIR_T_get_num_pvars(num_pvar);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
-fn_exit:
-    return mpi_errno;
-fn_fail:
-    goto fn_exit;
-}
-
 #endif /* MPICH_MPI_FROM_PMPI */
 
 #undef FUNCNAME
@@ -63,48 +42,22 @@ Output Parameters:
 int MPI_T_pvar_get_num(int *num_pvar)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_GET_NUM);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_GET_NUM);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_PVAR_GET_NUM);
 
-    /* Validate parameters, especially handles needing to be converted */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            MPIR_ERRTEST_ARGNULL(num_pvar, "num_pvar", mpi_errno);
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
     /* ... body of routine ...  */
 
-    mpi_errno = MPIR_T_pvar_get_num_impl(num_pvar);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    if (num_pvar != NULL)
+        *num_pvar = utarray_len(pvar_table);
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_PVAR_GET_NUM);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/pvar_handle_alloc.c b/src/mpi_t/pvar_handle_alloc.c
index e61ce49..151ca78 100644
--- a/src/mpi_t/pvar_handle_alloc.c
+++ b/src/mpi_t/pvar_handle_alloc.c
@@ -23,9 +23,9 @@
 #undef MPI_T_pvar_handle_alloc
 #define MPI_T_pvar_handle_alloc PMPI_T_pvar_handle_alloc
 
-/* define storage for the ALL_HANDLES constant */
-struct MPIR_T_pvar_handle MPIR_T_pvar_all_handles_obj = {NULL, NULL, NULL, NULL, -1, -1, NULL, NULL};
-struct MPIR_T_pvar_handle * const MPI_T_PVAR_ALL_HANDLES = &MPIR_T_pvar_all_handles_obj;
+/* Define storage for the ALL_HANDLES constant */
+MPIR_T_pvar_handle_t MPIR_T_pvar_all_handles_obj = {0};
+MPIR_T_pvar_handle_t * const MPI_T_PVAR_ALL_HANDLES = &MPIR_T_pvar_all_handles_obj;
 
 /* any non-MPI functions go here, especially non-static ones */
 
@@ -33,38 +33,124 @@ struct MPIR_T_pvar_handle * const MPI_T_PVAR_ALL_HANDLES = &MPIR_T_pvar_all_hand
 #define FUNCNAME MPIR_T_pvar_handle_alloc_impl
 #undef FCNAME
 #define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPIR_T_pvar_handle_alloc_impl(MPI_T_pvar_session session, int pvar_index, void *obj_handle, MPI_T_pvar_handle *handle, int *count)
+int MPIR_T_pvar_handle_alloc_impl(MPI_T_pvar_session session, int pvar_index,
+                                  void *obj_handle, MPI_T_pvar_handle *handle,int *count)
 {
     int mpi_errno = MPI_SUCCESS;
-    struct MPIR_T_pvar_info *info = NULL;
+    int cnt, extra, bytes;
+    int is_sum, is_watermark;
+    const pvar_table_entry_t *info;
+    MPIR_T_pvar_handle_t *hnd;
+
     MPIU_CHKPMEM_DECL(1);
 
-    *handle = MPI_T_PVAR_HANDLE_NULL;
-    *count = 0xdeadbeef;
+    info = (pvar_table_entry_t *) utarray_eltptr(pvar_table, pvar_index);
 
-    mpi_errno = MPIR_T_get_pvar_info_by_idx(pvar_index, &info);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    if (info->get_count == NULL) {
+        cnt = info->count;
+    } else {
+        mpi_errno = info->get_count(info->addr, obj_handle, &cnt);
+        if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    }
 
-    MPIU_CHKPMEM_MALLOC(*handle, MPI_T_pvar_handle, sizeof(**handle), mpi_errno, "performance var handle");
+    bytes = MPID_Datatype_get_basic_size(info->datatype);
+    is_sum = FALSE;
+    is_watermark = FALSE;
+    extra = 0;
 
-    /* setup the common fields */
-    (*handle)->session = session;
-    (*handle)->info = info;
+    if (info->varclass == MPI_T_PVAR_CLASS_COUNTER ||
+        info->varclass == MPI_T_PVAR_CLASS_AGGREGATE ||
+        info->varclass == MPI_T_PVAR_CLASS_TIMER)
+    {
+        /* Extra memory for accum, offset, current */
+        is_sum = TRUE;
+        extra = bytes * cnt * 3;
+    } else if (info->varclass == MPI_T_PVAR_CLASS_HIGHWATERMARK ||
+               info->varclass == MPI_T_PVAR_CLASS_LOWWATERMARK)
+    {
+        is_watermark = TRUE;
+    }
 
-    /* now some defaults that the create_fn might override */
-    (*handle)->bytes = -1;
-    (*handle)->vtable = NULL;
-    (*handle)->handle_state = NULL;
-    (*handle)->free_handle_state = FALSE;
+    /* Allocate memory and bzero it */
+    MPIU_CHKPMEM_CALLOC(hnd, MPIR_T_pvar_handle_t*, sizeof(*hnd) + extra,
+                        mpi_errno, "performance variable handle");
+
+    /* Setup the common fields */
+    if (is_sum)
+        hnd->flags |= MPIR_T_PVAR_FLAG_SUM;
+    else if (is_watermark)
+        hnd->flags |= MPIR_T_PVAR_FLAG_WATERMARK;
+
+    hnd->addr = info->addr;
+    hnd->datatype = info->datatype;
+    hnd->count = cnt;
+    hnd->varclass = info->varclass;
+    hnd->flags = info->flags;
+    hnd->session = session;
+    hnd->info = info;
+    hnd->obj_handle = obj_handle;
+    hnd->get_value = info->get_value;
+    hnd->bytes = bytes;
+    hnd->count = cnt;
+
+    /* Init pointers to cache buffers for a SUM */
+    if (MPIR_T_pvar_is_sum(hnd)) {
+        hnd->accum = (char*)(hnd) + sizeof(*hnd);
+        hnd->offset = (char*)(hnd) + sizeof(*hnd) + bytes*cnt;
+        hnd->current = (char*)(hnd) + sizeof(*hnd) + bytes*cnt*2;
+    }
 
-    /* dispatch particulars of the handle to the creator function */
-    mpi_errno = info->create_fn(obj_handle, (*handle), count);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    if (MPIR_T_pvar_is_continuous(hnd))
+        MPIR_T_pvar_set_started(hnd);
+
+    /* Set starting value of a continuous SUM */
+    if (MPIR_T_pvar_is_continuous(hnd) && MPIR_T_pvar_is_sum(hnd)) {
+        /* Cache current value of a SUM in offset.
+         * accum is zero since we called CALLOC before.
+         */
+        if (hnd->get_value == NULL)
+            MPIU_Memcpy(hnd->offset, hnd->addr, bytes*cnt);
+        else
+            hnd->get_value(hnd->addr, hnd->obj_handle, hnd->count, hnd->offset);
+    }
 
-    /* some creator functions may not assign this */
-    (*handle)->count = *count;
+     /* Link a WATERMARK handle to its pvar & set starting value if continuous */
+    if (MPIR_T_pvar_is_watermark(hnd)) {
+        MPIR_T_pvar_watermark_t *mark = (MPIR_T_pvar_watermark_t *)hnd->addr;
+        if (!mark->first_used) {
+            /* Use the special handle slot for optimization if available */
+            mark->first_used = TRUE;
+            MPIR_T_pvar_set_first(hnd);
+
+            /* Set starting value */
+            if (MPIR_T_pvar_is_continuous(hnd)) {
+                mark->first_started = TRUE;
+                mark->watermark = mark->current;
+            } else {
+                mark->first_started = FALSE;
+            }
+        } else {
+            /* If the special handle slot is unavailable, link it to hlist */
+            if (mark->hlist == NULL) {
+                hnd->prev2 = hnd;
+                mark->hlist = hnd;
+            } else {
+                hnd->prev2 = hnd;
+                hnd->next2 = mark->hlist;
+                mark->hlist->prev2 = hnd;
+                mark->hlist = hnd;
+            }
+
+            /* Set starting value */
+            if (MPIR_T_pvar_is_continuous(hnd))
+                hnd->watermark = mark->current;
+        }
+    }
 
-    MPL_DL_APPEND(session->hlist, *handle);
+    /* Link the handle in its session and return it */
+    MPL_DL_APPEND(session->hlist, hnd);
+    *handle = hnd;
+    *count = cnt;
 
     MPIU_CHKPMEM_COMMIT();
 fn_exit:
@@ -98,37 +184,24 @@ Output Parameters:
 
 .N Errors
 @*/
-int MPI_T_pvar_handle_alloc(MPI_T_pvar_session session, int pvar_index, void *obj_handle, MPI_T_pvar_handle *handle, int *count)
+int MPI_T_pvar_handle_alloc(MPI_T_pvar_session session, int pvar_index,
+                            void *obj_handle, MPI_T_pvar_handle *handle, int *count)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_HANDLE_ALLOC);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_HANDLE_ALLOC);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_PVAR_HANDLE_ALLOC);
 
-    /* Validate parameters, especially handles needing to be converted */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
+    /* Validate parameters  */
 #   ifdef HAVE_ERROR_CHECKING
     {
         MPID_BEGIN_ERROR_CHECKS
         {
             MPIR_ERRTEST_ARGNULL(count, "count", mpi_errno);
             MPIR_ERRTEST_ARGNULL(handle, "handle", mpi_errno);
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
+            /* Do not test obj_handle since it may be NULL when no binding */
         }
         MPID_END_ERROR_CHECKS
     }
@@ -136,14 +209,26 @@ int MPI_T_pvar_handle_alloc(MPI_T_pvar_session session, int pvar_index, void *ob
 
     /* ... body of routine ...  */
 
+    pvar_table_entry_t *entry;
+    if (pvar_index < 0 || pvar_index >= utarray_len(pvar_table)) {
+        mpi_errno = MPI_T_ERR_INVALID_INDEX;
+        goto fn_fail;
+    }
+
+    entry = (pvar_table_entry_t *) utarray_eltptr(pvar_table, pvar_index);
+    if (!entry->active) {
+        mpi_errno = MPI_T_ERR_INVALID_INDEX;
+        goto fn_fail;
+    }
+
     mpi_errno = MPIR_T_pvar_handle_alloc_impl(session, pvar_index, obj_handle, handle, count);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    if (mpi_errno) goto fn_fail;
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_PVAR_HANDLE_ALLOC);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/pvar_handle_free.c b/src/mpi_t/pvar_handle_free.c
index 68b687d..d22df03 100644
--- a/src/mpi_t/pvar_handle_free.c
+++ b/src/mpi_t/pvar_handle_free.c
@@ -32,15 +32,33 @@
 int MPIR_T_pvar_handle_free_impl(MPI_T_pvar_session session, MPI_T_pvar_handle *handle)
 {
     int mpi_errno = MPI_SUCCESS;
-
-    MPL_DL_DELETE(session->hlist, (*handle));
-
-    if ((*handle)->free_handle_state && (*handle)->handle_state != NULL) {
-        MPIU_Free((*handle)->handle_state);
+    MPIR_T_pvar_handle_t *hnd = *handle;
+
+    MPL_DL_DELETE(session->hlist, hnd);
+
+    /* Unlink handle from pvar if it is a watermark */
+    if (MPIR_T_pvar_is_watermark(hnd)) {
+        MPIR_T_pvar_watermark_t *mark = (MPIR_T_pvar_watermark_t *)hnd->addr;
+        if (MPIR_T_pvar_is_first(hnd)) {
+            mark->first_used = 0;
+            mark->first_started = 0;
+        } else {
+            MPIU_Assert(mark->hlist);
+            if (mark->hlist == hnd) {
+                /* hnd happens to be the head */
+                mark->hlist = hnd->next2;
+                if (mark->hlist != NULL)
+                    mark->hlist->prev2 = mark->hlist;
+            } else {
+                hnd->prev2->next2 = hnd->next2;
+                if (hnd->next2 != NULL)
+                    hnd->next2->prev2 = hnd->prev2;
+            }
+        }
     }
-    MPIU_Free(*handle);
 
-    (*handle) = MPI_T_PVAR_HANDLE_NULL;
+    MPIU_Free(hnd);
+    *handle = MPI_T_PVAR_HANDLE_NULL;
 
 fn_exit:
     return mpi_errno;
@@ -70,9 +88,10 @@ Input/Output Parameters:
 int MPI_T_pvar_handle_free(MPI_T_pvar_session session, MPI_T_pvar_handle *handle)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_HANDLE_FREE);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_HANDLE_FREE);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_PVAR_HANDLE_FREE);
 
     /* Validate parameters, especially handles needing to be converted */
@@ -80,22 +99,8 @@ int MPI_T_pvar_handle_free(MPI_T_pvar_session session, MPI_T_pvar_handle *handle
     {
         MPID_BEGIN_ERROR_CHECKS
         {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
+            MPIR_ERRTEST_ARGNULL(session, "session", mpi_errno);
+            MPIR_ERRTEST_ARGNULL(handle, "handle", mpi_errno);
             if (mpi_errno != MPI_SUCCESS) goto fn_fail;
         }
         MPID_END_ERROR_CHECKS
@@ -103,6 +108,10 @@ int MPI_T_pvar_handle_free(MPI_T_pvar_session session, MPI_T_pvar_handle *handle
 #   endif /* HAVE_ERROR_CHECKING */
 
     /* ... body of routine ...  */
+    if ((*handle) == MPI_T_PVAR_ALL_HANDLES || (*handle)->session != session) {
+        mpi_errno = MPI_T_ERR_INVALID_HANDLE;
+        goto fn_fail;
+    }
 
     mpi_errno = MPIR_T_pvar_handle_free_impl(session, handle);
     if (mpi_errno) MPIU_ERR_POP(mpi_errno);
@@ -111,7 +120,7 @@ int MPI_T_pvar_handle_free(MPI_T_pvar_session session, MPI_T_pvar_handle *handle
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_PVAR_HANDLE_FREE);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/pvar_read.c b/src/mpi_t/pvar_read.c
index 34d7041..6035476 100644
--- a/src/mpi_t/pvar_read.c
+++ b/src/mpi_t/pvar_read.c
@@ -28,17 +28,172 @@
 #define FUNCNAME MPIR_T_pvar_read_impl
 #undef FCNAME
 #define FCNAME MPIU_QUOTE(FUNCNAME)
-int MPIR_T_pvar_read_impl(MPI_T_pvar_session session, MPI_T_pvar_handle handle, void *buf)
+int MPIR_T_pvar_read_impl(MPI_T_pvar_session session, MPI_T_pvar_handle handle, void *restrict buf)
 {
-    int mpi_errno = MPI_SUCCESS;
+    int i, mpi_errno = MPI_SUCCESS;
+    void *tmp;
+
+    /* Reading a never started pvar, or a stopped and then reset wartermark,
+     * will run into this nasty situation. Wait for an error code to be defined
+     * by the Standard. Currently, use MPI_ERR_OTHER.
+     */
+    if (!MPIR_T_pvar_is_oncestarted(handle)) {
+        mpi_errno = MPI_ERR_OTHER;
+        goto fn_fail;
+    }
 
-    /* the extra indirection through "info" might be too costly for some tools,
-     * consider moving this value to or caching it in the handle itself */
-    if (likely(handle->info->impl_kind == MPIR_T_PVAR_IMPL_SIMPLE)) {
-        MPIU_Memcpy(buf, handle->handle_state, handle->count * handle->bytes);
+    /* For SUM pvars, return accum value + current value - offset value */
+    if (MPIR_T_pvar_is_sum(handle) && MPIR_T_pvar_is_started(handle)) {
+        if (handle->get_value == NULL) {
+            /* A running SUM without callback. Read its current value directly */
+            switch (handle->datatype) {
+            case MPI_UNSIGNED_LONG_LONG:
+                /* Put long long and double at front, since they are common */
+                for (i = 0; i < handle->count; i++) {
+                    ((unsigned long long *)buf)[i] =
+                        ((unsigned long long *)handle->accum)[i]
+                        + ((unsigned long long *)handle->addr)[i]
+                        - ((unsigned long long *)handle->offset)[i];
+                }
+                break;
+            case MPI_DOUBLE:
+                for (i = 0; i < handle->count; i++) {
+                    ((double *)buf)[i] =
+                        ((double *)handle->accum)[i]
+                        + ((double *)handle->addr)[i]
+                        - ((double *)handle->offset)[i];
+                }
+                break;
+            case MPI_UNSIGNED:
+                for (i = 0; i < handle->count; i++) {
+                    ((unsigned *)buf)[i] =
+                        ((unsigned *)handle->accum)[i]
+                        + ((unsigned *)handle->addr)[i]
+                        - ((unsigned *)handle->offset)[i];
+                }
+                break;
+            case MPI_UNSIGNED_LONG:
+                for (i = 0; i < handle->count; i++) {
+                   ((unsigned long *)buf)[i] =
+                        ((unsigned long *)handle->accum)[i]
+                        + ((unsigned long *)handle->addr)[i]
+                        - ((unsigned long *)handle->offset)[i];
+                }
+                break;
+            default:
+                /* Code should never come here */
+                mpi_errno = MPI_ERR_INTERN; goto fn_fail;
+                break;
+            }
+        } else {
+            /* A running SUM with callback. Read its current value into handle */
+            handle->get_value(handle->addr, handle->obj_handle,
+                              handle->count, handle->current);
+
+            switch (handle->datatype) {
+            case MPI_UNSIGNED_LONG_LONG:
+                for (i = 0; i < handle->count; i++) {
+                    ((unsigned long long *)buf)[i] =
+                        ((unsigned long long *)handle->accum)[i]
+                        + ((unsigned long *)handle->current)[i]
+                        - ((unsigned long long *)handle->offset)[i];
+                }
+                break;
+            case MPI_DOUBLE:
+                for (i = 0; i < handle->count; i++) {
+                    ((double *)buf)[i] =
+                        ((double *)handle->accum)[i]
+                        + ((double *)handle->current)[i]
+                        - ((double *)handle->offset)[i];
+                }
+                break;
+            case MPI_UNSIGNED:
+                for (i = 0; i < handle->count; i++) {
+                    ((unsigned *)buf)[i] =
+                        ((unsigned *)handle->accum)[i]
+                        + ((unsigned *)handle->current)[i]
+                        - ((unsigned *)handle->offset)[i];
+                }
+                break;
+            case MPI_UNSIGNED_LONG:
+                for (i = 0; i < handle->count; i++) {
+                    ((unsigned long *)buf)[i] =
+                        ((unsigned long *)handle->accum)[i]
+                        + ((unsigned long *)handle->current)[i]
+                        - ((unsigned long *)handle->offset)[i];
+                }
+                break;
+            default:
+                /* Code should never come here */
+                mpi_errno = MPI_ERR_INTERN; goto fn_fail;
+                break;
+            }
+        }
     }
-    else {
-        MPIU_Assertp(FALSE); /* _IMPL_CB not yet implemented */
+    else if (MPIR_T_pvar_is_sum(handle) && !MPIR_T_pvar_is_started(handle)) {
+        /* A SUM is stopped. Return accum directly */
+        MPIU_Memcpy(buf, handle->accum, handle->bytes * handle->count);
+    }
+    else if (MPIR_T_pvar_is_watermark(handle)) {
+        /* Callback and array are not allowed for watermarks, since they
+         * can not gurantee correct semantics of watermarks.
+         */
+        MPIU_Assert(handle->get_value == NULL && handle->count == 1);
+
+        if (MPIR_T_pvar_is_first(handle)) {
+            /* Current value of the first handle of a watermark is stored at
+             * a special location nearby the watermark itself.
+             */
+            switch (handle->datatype) {
+            case MPI_UNSIGNED_LONG_LONG:
+                 *(unsigned long long *)buf =
+                    ((MPIR_T_pvar_watermark_t *)handle->addr)->watermark.ull;
+                 break;
+            case MPI_DOUBLE:
+                 *(double *)buf =
+                    ((MPIR_T_pvar_watermark_t *)handle->addr)->watermark.f;
+                 break;
+            case MPI_UNSIGNED:
+                 *(unsigned *)buf =
+                    ((MPIR_T_pvar_watermark_t *)handle->addr)->watermark.u;
+                 break;
+            case MPI_UNSIGNED_LONG:
+                 *(unsigned long *)buf =
+                    ((MPIR_T_pvar_watermark_t *)handle->addr)->watermark.ul;
+                 break;
+            default:
+                /* Code should never come here */
+                 mpi_errno = MPI_ERR_INTERN; goto fn_fail;
+                 break;
+            }
+        } else {
+            /* For remaining handles, their current value are in the handle */
+            switch (handle->datatype) {
+            case MPI_UNSIGNED_LONG_LONG:
+                 *(unsigned long long *)buf = handle->watermark.ull;
+                 break;
+            case MPI_DOUBLE:
+                 *(double *)buf = handle->watermark.f;
+                 break;
+            case MPI_UNSIGNED:
+                 *(unsigned *)buf = handle->watermark.u;
+                 break;
+            case MPI_UNSIGNED_LONG:
+                 *(unsigned long *)buf = handle->watermark.ul;
+                 break;
+            default:
+                /* Code should never come here */
+                 mpi_errno = MPI_ERR_INTERN; goto fn_fail;
+                 break;
+            }
+        }
+    } else {
+        /* For STATE, LEVEL, SIZE, PERCENTAGE, no caching is needed */
+        if (handle->get_value == NULL)
+            MPIU_Memcpy(buf, handle->addr, handle->bytes * handle->count);
+        else
+            handle->get_value(handle->addr, handle->obj_handle,
+                              handle->count, buf);
     }
 
 fn_exit:
@@ -72,48 +227,42 @@ Output Parameters:
 int MPI_T_pvar_read(MPI_T_pvar_session session, MPI_T_pvar_handle handle, void *buf)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_READ);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_READ);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_PVAR_READ);
 
-    /* Validate parameters, especially handles needing to be converted */
+    /* Validate parameters */
 #   ifdef HAVE_ERROR_CHECKING
     {
         MPID_BEGIN_ERROR_CHECKS
         {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            MPIR_ERRTEST_ARGNULL(session, "session", mpi_errno);
+            MPIR_ERRTEST_ARGNULL(handle, "handle", mpi_errno);
+            MPIR_ERRTEST_ARGNULL(buf, "buf", mpi_errno);
         }
         MPID_END_ERROR_CHECKS
     }
 #   endif /* HAVE_ERROR_CHECKING */
 
-    /* Convert MPI object handles to object pointers */
+    /* ... body of routine ...  */
 
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
+    if (handle == MPI_T_PVAR_ALL_HANDLES  || session != handle->session
+        || !MPIR_T_pvar_is_oncestarted(handle))
     {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
+        mpi_errno = MPI_T_ERR_INVALID_HANDLE;
+        goto fn_fail;
     }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* ... body of routine ...  */
 
     mpi_errno = MPIR_T_pvar_read_impl(session, handle, buf);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    if (mpi_errno) goto fn_fail;
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_PVAR_READ);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/pvar_readreset.c b/src/mpi_t/pvar_readreset.c
index 86fb6f5..2347218 100644
--- a/src/mpi_t/pvar_readreset.c
+++ b/src/mpi_t/pvar_readreset.c
@@ -30,9 +30,13 @@
 #define FCNAME MPIU_QUOTE(FUNCNAME)
 int MPIR_T_pvar_readreset_impl(MPI_T_pvar_session session, MPI_T_pvar_handle handle, void *buf)
 {
-    int mpi_errno = MPI_ERR_INTERN;
+    int mpi_errno = MPI_SUCCESS;
+
+    mpi_errno = MPIR_T_pvar_read_impl(session, handle, buf);
+    if (mpi_errno != MPI_SUCCESS) goto fn_fail;
 
-    /* TODO implement this function */
+    mpi_errno = MPIR_T_pvar_reset_impl(session, handle);
+    if (mpi_errno != MPI_SUCCESS) goto fn_fail;
 
 fn_exit:
     return mpi_errno;
@@ -65,48 +69,47 @@ Output Parameters:
 int MPI_T_pvar_readreset(MPI_T_pvar_session session, MPI_T_pvar_handle handle, void *buf)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_READRESET);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_READRESET);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_PVAR_READRESET);
 
-    /* Validate parameters, especially handles needing to be converted */
+    /* Validate parameters */
 #   ifdef HAVE_ERROR_CHECKING
     {
         MPID_BEGIN_ERROR_CHECKS
         {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            MPIR_ERRTEST_ARGNULL(session, "session", mpi_errno);
+            MPIR_ERRTEST_ARGNULL(handle, "handle", mpi_errno);
+            MPIR_ERRTEST_ARGNULL(buf, "buf", mpi_errno);
         }
         MPID_END_ERROR_CHECKS
     }
 #   endif /* HAVE_ERROR_CHECKING */
 
-    /* Convert MPI object handles to object pointers */
+    /* ... body of routine ...  */
 
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
+    if (handle == MPI_T_PVAR_ALL_HANDLES  || session != handle->session
+        || !MPIR_T_pvar_is_oncestarted(handle))
     {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
+        mpi_errno = MPI_T_ERR_INVALID_HANDLE;
+        goto fn_fail;
     }
-#   endif /* HAVE_ERROR_CHECKING */
 
-    /* ... body of routine ...  */
+    if (!MPIR_T_pvar_is_atomic(handle)) {
+        mpi_errno = MPI_T_ERR_PVAR_NO_ATOMIC;
+        goto fn_fail;
+    }
 
     mpi_errno = MPIR_T_pvar_readreset_impl(session, handle, buf);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    if (mpi_errno != MPI_SUCCESS) goto fn_fail;
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_PVAR_READRESET);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/pvar_reset.c b/src/mpi_t/pvar_reset.c
index 1e2fc72..783ccdd 100644
--- a/src/mpi_t/pvar_reset.c
+++ b/src/mpi_t/pvar_reset.c
@@ -5,6 +5,7 @@
  */
 
 #include "mpiimpl.h"
+#include "mpl_utlist.h"
 
 /* -- Begin Profiling Symbol Block for routine MPI_T_pvar_reset */
 #if defined(HAVE_PRAGMA_WEAK)
@@ -30,9 +31,39 @@
 #define FCNAME MPIU_QUOTE(FUNCNAME)
 int MPIR_T_pvar_reset_impl(MPI_T_pvar_session session, MPI_T_pvar_handle handle)
 {
-    int mpi_errno = MPI_ERR_INTERN;
-
-    /* TODO implement this function */
+    int mpi_errno = MPI_SUCCESS;
+    MPIR_T_pvar_watermark_t *mark;
+
+    if (MPIR_T_pvar_is_sum(handle)) {
+        /* Use zero as starting value */
+        memset(handle->accum, 0, handle->bytes * handle->count);
+
+        /* Record current value as offset when pvar is running (i.e., started) */
+        if (MPIR_T_pvar_is_started(handle)) {
+            if (handle->get_value == NULL) {
+                MPIU_Memcpy(handle->offset, handle->addr, handle->bytes * handle->count);
+            } else {
+                handle->get_value(handle->addr, handle->obj_handle,
+                                  handle->count, handle->offset);
+            }
+        }
+    } else if (MPIR_T_pvar_is_watermark(handle)) {
+        if (MPIR_T_pvar_is_started(handle)) {
+            /* Use the current value as starting value when pvar is running */
+            mark = (MPIR_T_pvar_watermark_t *)handle->addr;
+            if (MPIR_T_pvar_is_first(handle)) {
+                MPIU_Assert(mark->first_used);
+                mark->watermark = mark->current;
+            } else {
+                handle->watermark = mark->current;
+            }
+        } else {
+            /* If pvar is stopped, clear the oncestarted flag
+             * so that when it is re-started, it looks new.
+             */
+            MPIR_T_pvar_unset_oncestarted(handle);
+        }
+    }
 
 fn_exit:
     return mpi_errno;
@@ -62,48 +93,57 @@ Input Parameters:
 int MPI_T_pvar_reset(MPI_T_pvar_session session, MPI_T_pvar_handle handle)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_RESET);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_RESET);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_PVAR_RESET);
 
-    /* Validate parameters, especially handles needing to be converted */
+     /* Validate parameters, especially handles needing to be converted */
 #   ifdef HAVE_ERROR_CHECKING
     {
         MPID_BEGIN_ERROR_CHECKS
         {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            MPIR_ERRTEST_ARGNULL(session, "session", mpi_errno);
+            MPIR_ERRTEST_ARGNULL(handle,  "handle",  mpi_errno);
         }
         MPID_END_ERROR_CHECKS
     }
 #   endif /* HAVE_ERROR_CHECKING */
 
-    /* Convert MPI object handles to object pointers */
+    /* ... body of routine ...  */
 
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+    /* If handle is MPI_T_PVAR_ALL_HANDLES, dispatch the call.
+     * Otherwise, do correctness check, then go to impl.
+     */
+    MPIR_T_pvar_handle_t *hnd;
+    if (handle == MPI_T_PVAR_ALL_HANDLES) {
+        MPL_DL_FOREACH(session->hlist, hnd) {
+            if (!MPIR_T_pvar_is_readonly(hnd)) {
+                mpi_errno = MPIR_T_pvar_reset_impl(session, hnd);
+                if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            }
+        }
+    } else {
+        if (handle->session != session) {
+            mpi_errno = MPI_T_ERR_INVALID_HANDLE;
+            goto fn_fail;
         }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
 
-    /* ... body of routine ...  */
+        if (MPIR_T_pvar_is_readonly(handle)) {
+            mpi_errno = MPI_T_ERR_PVAR_NO_WRITE;
+            goto fn_fail;
+        }
 
-    mpi_errno = MPIR_T_pvar_reset_impl(session, handle);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+        mpi_errno = MPIR_T_pvar_reset_impl(session, handle);
+        if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+    }
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_PVAR_RESET);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/pvar_session_create.c b/src/mpi_t/pvar_session_create.c
index 836ef45..ec5f5b7 100644
--- a/src/mpi_t/pvar_session_create.c
+++ b/src/mpi_t/pvar_session_create.c
@@ -69,9 +69,10 @@ Output Parameters:
 int MPI_T_pvar_session_create(MPI_T_pvar_session *session)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_SESSION_CREATE);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_SESSION_CREATE);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_PVAR_SESSION_CREATE);
 
     /* Validate parameters, especially handles needing to be converted */
@@ -79,23 +80,7 @@ int MPI_T_pvar_session_create(MPI_T_pvar_session *session)
     {
         MPID_BEGIN_ERROR_CHECKS
         {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            MPIR_ERRTEST_ARGNULL(session, "session", mpi_errno);
         }
         MPID_END_ERROR_CHECKS
     }
@@ -104,13 +89,13 @@ int MPI_T_pvar_session_create(MPI_T_pvar_session *session)
     /* ... body of routine ...  */
 
     mpi_errno = MPIR_T_pvar_session_create_impl(session);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    if (mpi_errno != MPI_SUCCESS) goto fn_fail;
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_PVAR_SESSION_CREATE);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/pvar_session_free.c b/src/mpi_t/pvar_session_free.c
index 774323a..968b1fa 100644
--- a/src/mpi_t/pvar_session_free.c
+++ b/src/mpi_t/pvar_session_free.c
@@ -32,7 +32,7 @@
 int MPIR_T_pvar_session_free_impl(MPI_T_pvar_session *session)
 {
     int mpi_errno = MPI_SUCCESS;
-    struct MPIR_T_pvar_handle *hnd, *tmp;
+    MPIR_T_pvar_handle_t *hnd, *tmp;
 
     /* The draft standard is a bit unclear about the behavior of this function
      * w.r.t. outstanding pvar handles.  A conservative (from the implementors
@@ -74,9 +74,10 @@ Input/Output Parameters:
 int MPI_T_pvar_session_free(MPI_T_pvar_session *session)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_SESSION_FREE);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_SESSION_FREE);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_PVAR_SESSION_FREE);
 
     /* Validate parameters, especially handles needing to be converted */
@@ -84,23 +85,7 @@ int MPI_T_pvar_session_free(MPI_T_pvar_session *session)
     {
         MPID_BEGIN_ERROR_CHECKS
         {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
-    }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* Convert MPI object handles to object pointers */
-
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            MPIR_ERRTEST_ARGNULL(session, "session", mpi_errno);
         }
         MPID_END_ERROR_CHECKS
     }
@@ -109,13 +94,13 @@ int MPI_T_pvar_session_free(MPI_T_pvar_session *session)
     /* ... body of routine ...  */
 
     mpi_errno = MPIR_T_pvar_session_free_impl(session);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    if (mpi_errno != MPI_SUCCESS) goto fn_fail;
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_PVAR_SESSION_FREE);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/pvar_start.c b/src/mpi_t/pvar_start.c
index 8472b6c..21037c8 100644
--- a/src/mpi_t/pvar_start.c
+++ b/src/mpi_t/pvar_start.c
@@ -5,6 +5,7 @@
  */
 
 #include "mpiimpl.h"
+#include "mpl_utlist.h" /* For MPL_DL_FOREACH */
 
 /* -- Begin Profiling Symbol Block for routine MPI_T_pvar_start */
 #if defined(HAVE_PRAGMA_WEAK)
@@ -30,9 +31,39 @@
 #define FCNAME MPIU_QUOTE(FUNCNAME)
 int MPIR_T_pvar_start_impl(MPI_T_pvar_session session, MPI_T_pvar_handle handle)
 {
-    int mpi_errno = MPI_ERR_INTERN;
+    int mpi_errno = MPI_SUCCESS;
+    MPIR_T_pvar_watermark_t *mark;
+
+    if (MPIR_T_pvar_is_sum(handle)) {
+        /* To start SUM, we only need to cache its current value into offset.
+         * If it has ever been started, accum holds correct value. Otherwise,
+         * accum is zero since handle allocation.
+         */
+        if (handle->get_value == NULL) {
+            MPIU_Memcpy(handle->offset, handle->addr, handle->bytes * handle->count);
+        } else {
+            handle->get_value(handle->addr, handle->obj_handle,
+                              handle->count, handle->offset);
+        }
+    } else if (MPIR_T_pvar_is_watermark(handle)) {
+        /* To start WATERMARK, if it has not ever been started,
+         * we need to set its starting value properly.
+         */
+        mark = (MPIR_T_pvar_watermark_t *)handle->addr;
+
+        if (MPIR_T_pvar_is_first(handle)) {
+            MPIU_Assert(mark->first_used);
+            mark->first_started = TRUE;
+            if (!MPIR_T_pvar_is_oncestarted(handle))
+                mark->watermark = mark->current;
+        } else {
+            if (!MPIR_T_pvar_is_oncestarted(handle))
+                handle->watermark = mark->current;
+        }
+    }
 
-    /* TODO implement this function */
+    /* OK, let's get started */
+    MPIR_T_pvar_set_started(handle);
 
 fn_exit:
     return mpi_errno;
@@ -62,9 +93,10 @@ Input Parameters:
 int MPI_T_pvar_start(MPI_T_pvar_session session, MPI_T_pvar_handle handle)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_START);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_START);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_PVAR_START);
 
     /* Validate parameters, especially handles needing to be converted */
@@ -72,38 +104,42 @@ int MPI_T_pvar_start(MPI_T_pvar_session session, MPI_T_pvar_handle handle)
     {
         MPID_BEGIN_ERROR_CHECKS
         {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            MPIR_ERRTEST_ARGNULL(session, "session", mpi_errno);
+            MPIR_ERRTEST_ARGNULL(handle, "handle", mpi_errno);
         }
         MPID_END_ERROR_CHECKS
     }
 #   endif /* HAVE_ERROR_CHECKING */
 
-    /* Convert MPI object handles to object pointers */
+    /* ... body of routine ...  */
 
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
+    if (handle == MPI_T_PVAR_ALL_HANDLES) {
+        MPIR_T_pvar_handle_t *hnd;
+        MPL_DL_FOREACH(session->hlist, hnd) {
+            if (!MPIR_T_pvar_is_continuous(hnd) && !MPIR_T_pvar_is_started(hnd))
+                mpi_errno = MPIR_T_pvar_start_impl(session, hnd);
+        }
+    } else {
+        if (handle->session != session) {
+            mpi_errno = MPI_T_ERR_INVALID_HANDLE;
+            goto fn_fail;
+        }
+        if (MPIR_T_pvar_is_continuous(handle)) {
+            mpi_errno = MPI_T_ERR_PVAR_NO_STARTSTOP;
+            goto fn_fail;
+        }
+
+        if (!MPIR_T_pvar_is_started(handle)) {
+            mpi_errno = MPIR_T_pvar_start_impl(session, handle);
             if (mpi_errno != MPI_SUCCESS) goto fn_fail;
         }
-        MPID_END_ERROR_CHECKS
     }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* ... body of routine ...  */
-
-    mpi_errno = MPIR_T_pvar_start_impl(session, handle);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_PVAR_START);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/pvar_stop.c b/src/mpi_t/pvar_stop.c
index 8cf85a3..3b1599d 100644
--- a/src/mpi_t/pvar_stop.c
+++ b/src/mpi_t/pvar_stop.c
@@ -5,6 +5,7 @@
  */
 
 #include "mpiimpl.h"
+#include "mpl_utlist.h"
 
 /* -- Begin Profiling Symbol Block for routine MPI_T_pvar_stop */
 #if defined(HAVE_PRAGMA_WEAK)
@@ -30,9 +31,65 @@
 #define FCNAME MPIU_QUOTE(FUNCNAME)
 int MPIR_T_pvar_stop_impl(MPI_T_pvar_session session, MPI_T_pvar_handle handle)
 {
-    int mpi_errno = MPI_ERR_INTERN;
+    int i, mpi_errno = MPI_SUCCESS;
+    MPIR_T_pvar_watermark_t *mark;
+
+    MPIR_T_pvar_unset_started(handle);
+
+    /* Side-effect when pvar is SUM or WATERMARK */
+    if (MPIR_T_pvar_is_sum(handle)) {
+        /* Read the current value first */
+        if (handle->get_value == NULL) {
+            MPIU_Memcpy(handle->current, handle->addr, handle->bytes * handle->count);
+        } else {
+            handle->get_value(handle->addr, handle->obj_handle,
+                              handle->count, handle->current);
+        }
+
+        /* Substract offset from current, and accumulate the result to accum */
+        switch (handle->datatype) {
+        case MPI_UNSIGNED_LONG_LONG:
+            for (i = 0; i < handle->count; i++) {
+                ((unsigned long long *)handle->accum)[i] +=
+                    ((unsigned long long *)handle->current)[i]
+                    - ((unsigned long long *)handle->offset)[i];
+            }
+            break;
+        case MPI_DOUBLE:
+            for (i = 0; i < handle->count; i++) {
+                ((double *)handle->accum)[i] +=
+                    ((double *)handle->current)[i]
+                    - ((double *)handle->offset)[i];
+            }
+            break;
+        case MPI_UNSIGNED:
+            for (i = 0; i < handle->count; i++) {
+                ((unsigned *)handle->accum)[i] +=
+                    ((unsigned *)handle->current)[i]
+                    - ((unsigned *)handle->offset)[i];
+            }
+            break;
+        case MPI_UNSIGNED_LONG:
+            for (i = 0; i < handle->count; i++) {
+                ((unsigned long *)handle->accum)[i] +=
+                    ((unsigned long *)handle->current)[i]
+                    - ((unsigned long *)handle->offset)[i];
+            }
+            break;
+        default:
+            /* Code should never come here */
+            mpi_errno = MPI_ERR_INTERN; goto fn_fail;
+            break;
+        }
 
-    /* TODO implement this function */
+    } else if (MPIR_T_pvar_is_watermark(handle)) {
+        /* When handle is first, clear the flag in pvar too */
+        if (MPIR_T_pvar_is_first(handle)) {
+            mark = (MPIR_T_pvar_watermark_t *)handle->addr;
+            MPIU_Assert(mark->first_used);
+            mark->first_started = FALSE;
+        }
+    }
 
 fn_exit:
     return mpi_errno;
@@ -62,9 +119,10 @@ Input Parameters:
 int MPI_T_pvar_stop(MPI_T_pvar_session session, MPI_T_pvar_handle handle)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_STOP);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_STOP);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_PVAR_STOP);
 
     /* Validate parameters, especially handles needing to be converted */
@@ -72,38 +130,47 @@ int MPI_T_pvar_stop(MPI_T_pvar_session session, MPI_T_pvar_handle handle)
     {
         MPID_BEGIN_ERROR_CHECKS
         {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            MPIR_ERRTEST_ARGNULL(session, "session", mpi_errno);
+            MPIR_ERRTEST_ARGNULL(handle,  "handle",  mpi_errno);
         }
         MPID_END_ERROR_CHECKS
     }
 #   endif /* HAVE_ERROR_CHECKING */
 
-    /* Convert MPI object handles to object pointers */
+    /* ... body of routine ...  */
 
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
+    /* If handle is MPI_T_PVAR_ALL_HANDLES, dispatch the call.
+     * Otherwise, do correctness check, then go to impl.
+     */
+    if (handle == MPI_T_PVAR_ALL_HANDLES) {
+        MPIR_T_pvar_handle_t *hnd;
+        MPL_DL_FOREACH(session->hlist, hnd) {
+            if (!MPIR_T_pvar_is_continuous(hnd) && MPIR_T_pvar_is_started(hnd)) {
+                mpi_errno = MPIR_T_pvar_stop_impl(session, hnd);
+                if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            }
+        }
+    } else {
+        if (handle->session != session) {
+            mpi_errno = MPI_T_ERR_INVALID_HANDLE;
+            goto fn_fail;
+        }
+        if (MPIR_T_pvar_is_continuous(handle)) {
+            mpi_errno = MPI_T_ERR_PVAR_NO_STARTSTOP;
+            goto fn_fail;
+        }
+
+        if (MPIR_T_pvar_is_started(handle)) {
+            mpi_errno = MPIR_T_pvar_stop_impl(session, handle);
             if (mpi_errno != MPI_SUCCESS) goto fn_fail;
         }
-        MPID_END_ERROR_CHECKS
     }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* ... body of routine ...  */
-
-    mpi_errno = MPIR_T_pvar_stop_impl(session, handle);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_PVAR_STOP);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:
diff --git a/src/mpi_t/pvar_write.c b/src/mpi_t/pvar_write.c
index 26f4ecb..bf9f568 100644
--- a/src/mpi_t/pvar_write.c
+++ b/src/mpi_t/pvar_write.c
@@ -30,23 +30,8 @@
 #define FCNAME MPIU_QUOTE(FUNCNAME)
 int MPIR_T_pvar_write_impl(MPI_T_pvar_session session, MPI_T_pvar_handle handle, void *buf)
 {
-    int mpi_errno = MPI_SUCCESS;
-
-    MPIU_Assert(!handle->info->readonly);
-
-    /* the extra indirection through "info" might be too costly for some tools,
-     * consider moving this value to or caching it in the handle itself */
-    if (likely(handle->info->impl_kind == MPIR_T_PVAR_IMPL_SIMPLE)) {
-        MPIU_Memcpy(handle->handle_state, buf, handle->count * handle->bytes);
-    }
-    else {
-        MPIU_Assertp(FALSE); /* _IMPL_CB not yet implemented */
-    }
-
-fn_exit:
-    return mpi_errno;
-fn_fail:
-    goto fn_exit;
+    /* This function should never be called */
+    return MPI_ERR_INTERN;
 }
 
 #endif /* MPICH_MPI_FROM_PMPI */
@@ -72,9 +57,10 @@ Input Parameters:
 int MPI_T_pvar_write(MPI_T_pvar_session session, MPI_T_pvar_handle handle, void *buf)
 {
     int mpi_errno = MPI_SUCCESS;
-    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_WRITE);
 
-    MPIU_THREAD_CS_ENTER(ALLFUNC,);
+    MPID_MPI_STATE_DECL(MPID_STATE_MPI_T_PVAR_WRITE);
+    MPIR_T_FAIL_IF_UNINITIALIZED();
+    MPIR_T_THREAD_CS_ENTER();
     MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_T_PVAR_WRITE);
 
     /* Validate parameters, especially handles needing to be converted */
@@ -82,38 +68,36 @@ int MPI_T_pvar_write(MPI_T_pvar_session session, MPI_T_pvar_handle handle, void
     {
         MPID_BEGIN_ERROR_CHECKS
         {
-
-            /* TODO more checks may be appropriate */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
+            MPIR_ERRTEST_ARGNULL(session, "session", mpi_errno);
+            MPIR_ERRTEST_ARGNULL(handle, "handle", mpi_errno);
+            MPIR_ERRTEST_ARGNULL(buf, "buf", mpi_errno);
         }
         MPID_END_ERROR_CHECKS
     }
 #   endif /* HAVE_ERROR_CHECKING */
 
-    /* Convert MPI object handles to object pointers */
+    /* ... body of routine ...  */
 
-    /* Validate parameters and objects (post conversion) */
-#   ifdef HAVE_ERROR_CHECKING
-    {
-        MPID_BEGIN_ERROR_CHECKS
-        {
-            /* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
-            if (mpi_errno != MPI_SUCCESS) goto fn_fail;
-        }
-        MPID_END_ERROR_CHECKS
+    if (handle == MPI_T_PVAR_ALL_HANDLES || handle->session != session) {
+        mpi_errno = MPI_T_ERR_INVALID_HANDLE;
+        goto fn_fail;
+    } if (MPIR_T_pvar_is_readonly(handle)) {
+        mpi_errno = MPI_T_ERR_PVAR_NO_WRITE;
+        goto fn_fail;
+    } else {
+        /* We should never expose writable pvars */
+        mpi_errno = MPI_ERR_INTERN;
+        goto fn_fail;
     }
-#   endif /* HAVE_ERROR_CHECKING */
-
-    /* ... body of routine ...  */
 
     mpi_errno = MPIR_T_pvar_write_impl(session, handle, buf);
-    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+    if (mpi_errno != MPI_SUCCESS) goto fn_fail;
 
     /* ... end of body of routine ... */
 
 fn_exit:
     MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_T_PVAR_WRITE);
-    MPIU_THREAD_CS_EXIT(ALLFUNC,);
+    MPIR_T_THREAD_CS_EXIT();
     return mpi_errno;
 
 fn_fail:

-----------------------------------------------------------------------

Summary of changes:
 .gitignore                                         |    4 +-
 autogen.sh                                         |    4 +-
 configure.ac                                       |   22 +-
 maint/Makefile.mk                                  |    2 +-
 maint/gencvars                                     |  561 ++++++++
 maint/genparams                                    |  632 ---------
 src/include/Makefile.mk                            |    6 +-
 src/include/mpi.h.in                               |   42 +-
 src/include/mpiimpl.h                              |  130 +--
 src/include/mpit.h                                 |  623 +++++++++
 src/include/mpitimpl.h                             | 1404 ++++++++++++++++++++
 src/include/mpiutil.h                              |    6 +-
 src/mpi/coll/allgather.c                           |    4 +-
 src/mpi/coll/allgatherv.c                          |    8 +-
 src/mpi/coll/allred_group.c                        |    2 +-
 src/mpi/coll/allreduce.c                           |    8 +-
 src/mpi/coll/alltoall.c                            |    6 +-
 src/mpi/coll/alltoallv.c                           |    2 +-
 src/mpi/coll/alltoallw.c                           |    2 +-
 src/mpi/coll/barrier.c                             |    2 +-
 src/mpi/coll/bcast.c                               |   22 +-
 src/mpi/coll/gather.c                              |   16 +-
 src/mpi/coll/gatherv.c                             |    4 +-
 src/mpi/coll/helper_fns.c                          |   18 +-
 src/mpi/coll/iallgather.c                          |    4 +-
 src/mpi/coll/iallgatherv.c                         |    8 +-
 src/mpi/coll/iallreduce.c                          |    4 +-
 src/mpi/coll/ialltoall.c                           |    6 +-
 src/mpi/coll/ialltoallv.c                          |    2 +-
 src/mpi/coll/ialltoallw.c                          |    2 +-
 src/mpi/coll/ibcast.c                              |   10 +-
 src/mpi/coll/igather.c                             |   16 +-
 src/mpi/coll/igatherv.c                            |    4 +-
 src/mpi/coll/ired_scat.c                           |    4 +-
 src/mpi/coll/ired_scat_block.c                     |    4 +-
 src/mpi/coll/ireduce.c                             |    4 +-
 src/mpi/coll/iscatter.c                            |    2 +-
 src/mpi/coll/red_scat.c                            |    4 +-
 src/mpi/coll/red_scat_block.c                      |    4 +-
 src/mpi/coll/reduce.c                              |    8 +-
 src/mpi/coll/scatter.c                             |    2 +-
 src/mpi/comm/comm_split.c                          |    2 +-
 src/mpi/comm/commutil.c                            |    4 +-
 src/mpi/debugger/dbginit.c                         |    4 +-
 src/mpi/errhan/errnames.txt                        |    6 +-
 src/mpi/errhan/errutil.c                           |   22 +-
 src/mpi/init/finalize.c                            |   16 +-
 src/mpi/init/init.c                                |   17 +-
 src/mpi/init/initthread.c                          |   11 +-
 src/mpi_t/Makefile.mk                              |    6 +-
 src/mpi_t/cat_changed.c                            |   57 +-
 src/mpi_t/cat_get_categories.c                     |   44 +-
 src/mpi_t/cat_get_cvars.c                          |   44 +-
 src/mpi_t/cat_get_info.c                           |   74 +-
 src/mpi_t/cat_get_num.c                            |   58 +-
 src/mpi_t/cat_get_pv.c                             |  124 --
 src/mpi_t/cat_get_pvars.c                          |  124 ++
 src/mpi_t/cvar_get_info.c                          |  121 +--
 src/mpi_t/cvar_get_num.c                           |   57 +-
 src/mpi_t/cvar_handle_alloc.c                      |   68 +-
 src/mpi_t/cvar_handle_free.c                       |   61 +-
 src/mpi_t/cvar_read.c                              |  101 +-
 src/mpi_t/cvar_write.c                             |  111 +-
 src/mpi_t/enum_get_info.c                          |   56 +-
 src/mpi_t/enum_get_item.c                          |   66 +-
 src/mpi_t/mpi_t_util.c                             |  151 ---
 src/mpi_t/mpit.c                                   |  467 +++++++
 src/mpi_t/mpit_finalize.c                          |  190 ++-
 src/mpi_t/mpit_init_thread.c                       |  145 --
 src/mpi_t/mpit_initthread.c                        |  144 ++
 src/mpi_t/pvar_get_info.c                          |  116 +-
 src/mpi_t/pvar_get_num.c                           |   59 +-
 src/mpi_t/pvar_handle_alloc.c                      |  177 ++-
 src/mpi_t/pvar_handle_free.c                       |   61 +-
 src/mpi_t/pvar_read.c                              |  205 +++-
 src/mpi_t/pvar_readreset.c                         |   45 +-
 src/mpi_t/pvar_reset.c                             |   88 +-
 src/mpi_t/pvar_session_create.c                    |   27 +-
 src/mpi_t/pvar_session_free.c                      |   29 +-
 src/mpi_t/pvar_start.c                             |   80 +-
 src/mpi_t/pvar_stop.c                              |  111 ++-
 src/mpi_t/pvar_write.c                             |   58 +-
 .../ch3/channels/nemesis/include/mpid_nem_inline.h |    7 +-
 .../nemesis/include/mpidi_nem_statistics.h         |    5 +-
 .../channels/nemesis/netmod/portals4/ptl_init.c    |    2 +-
 .../ch3/channels/nemesis/netmod/tcp/tcp_init.c     |   16 +-
 .../channels/nemesis/netmod/wintcp/wintcp_init.c   |    4 +-
 src/mpid/ch3/channels/nemesis/src/ch3_progress.c   |    2 +-
 src/mpid/ch3/channels/nemesis/src/ch3i_comm.c      |    1 +
 src/mpid/ch3/channels/nemesis/src/mpid_nem_ckpt.c  |    2 +-
 .../ch3/channels/nemesis/src/mpid_nem_finalize.c   |    2 +-
 src/mpid/ch3/channels/nemesis/src/mpid_nem_init.c  |   61 +-
 .../ch3/channels/nemesis/src/mpid_nem_lmt_dma.c    |    2 +-
 .../ch3/channels/nemesis/src/mpid_nem_network.c    |    8 +-
 src/mpid/ch3/include/mpidi_common_statistics.h     |   71 -
 src/mpid/ch3/include/mpidi_recvq_statistics.h      |    5 +-
 src/mpid/ch3/src/ch3u_eager.c                      |    2 +-
 src/mpid/ch3/src/ch3u_handle_connection.c          |    1 +
 src/mpid/ch3/src/ch3u_handle_recv_pkt.c            |    2 +-
 src/mpid/ch3/src/ch3u_recvq.c                      |  306 ++---
 src/mpid/ch3/src/ch3u_rma_sync.c                   |   18 +-
 src/mpid/ch3/src/mpid_vc.c                         |    6 +-
 src/mpid/common/sock/iocp/sock.c                   |    8 +-
 src/mpid/common/sock/mpidu_sock.h                  |    2 +-
 src/mpid/pamid/src/mpidi_env.c                     |    2 +-
 src/nameserv/file/file_nameserv.c                  |    2 +-
 src/pm/hydra/ui/mpich/mpiexec.c                    |    2 +-
 src/pm/hydra/ui/mpich/utils.c                      |    6 +-
 src/util/Makefile.mk                               |    2 +-
 src/util/cvar/Makefile.mk                          |   17 +
 src/util/cvar/cvars.yml                            |  770 +++++++++++
 src/util/mem/handlemem.c                           |    2 +-
 src/util/param/Makefile.mk                         |   17 -
 src/util/param/params.yml                          |  656 ---------
 test/mpi/.gitignore                                |    1 +
 test/mpi/configure.ac                              |    1 +
 .../mpi_t/nem_fbox_fallback_to_queue_count.c       |   10 +-
 .../manual/mpi_t/unexpected_recvq_buffer_size.c    |   30 +-
 test/mpi/mpi_t/mpit_vars.c                         |   49 +-
 test/mpi/mpi_t/testlist                            |    2 +-
 test/mpi/threads/Makefile.am                       |    4 +-
 test/mpi/threads/mpi_t/Makefile.am                 |   13 +
 test/mpi/threads/mpi_t/mpit_threading.c            |  643 +++++++++
 test/mpi/threads/mpi_t/testlist                    |    1 +
 test/mpi/threads/testlist.in                       |    1 +
 125 files changed, 6327 insertions(+), 3462 deletions(-)
 create mode 100755 maint/gencvars
 delete mode 100755 maint/genparams
 create mode 100644 src/include/mpit.h
 create mode 100644 src/include/mpitimpl.h
 delete mode 100644 src/mpi_t/cat_get_pv.c
 create mode 100644 src/mpi_t/cat_get_pvars.c
 delete mode 100644 src/mpi_t/mpi_t_util.c
 create mode 100644 src/mpi_t/mpit.c
 delete mode 100644 src/mpi_t/mpit_init_thread.c
 create mode 100644 src/mpi_t/mpit_initthread.c
 delete mode 100644 src/mpid/ch3/include/mpidi_common_statistics.h
 create mode 100644 src/util/cvar/Makefile.mk
 create mode 100644 src/util/cvar/cvars.yml
 delete mode 100644 src/util/param/Makefile.mk
 delete mode 100644 src/util/param/params.yml
 create mode 100644 test/mpi/threads/mpi_t/Makefile.am
 create mode 100644 test/mpi/threads/mpi_t/mpit_threading.c
 create mode 100644 test/mpi/threads/mpi_t/testlist


hooks/post-receive
-- 
MPICH primary repository


More information about the commits mailing list