parflow-pdaf vs. parflow-master

CMakeLists.txt

@@ -14,6 +14,7 @@ set (CMAKE_C_STANDARD 11)
 
 include (CheckCCompilerFlag)
 include (CheckCXXCompilerFlag)
+include(GNUInstallDirs)
 
 # Check if compiler supports extra debugging flags
 foreach (DEBUG_FLAG "-ggdb" "-g3")
@@ -727,6 +728,10 @@ if ( ${PARFLOW_HAVE_CLM} )
   configure_file (pfsimulator/clm/parflow_config.F90.in ${PROJECT_BINARY_DIR}/pfsimulator/clm/parflow_config.F90)
 endif ( ${PARFLOW_HAVE_CLM} )
 
+# Install headers
+install(FILES ${CMAKE_BINARY_DIR}/include/parflow_config.h 
+              ${CMAKE_BINARY_DIR}/include/pfversion.h
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/parflow)
 #-----------------------------------------------------------------------------
 # CMAKE Subdirectories
 #-----------------------------------------------------------------------------

README.md

@@ -1,3 +1,14 @@
+# Fork of ParFlow pre-patched for TSMP-PDAF
+
+**DO NOT USE WITHOUT PDAF**
+
+This is a fork of ParFlow with changes added to run ParFlow in the
+TSMP-PDAF framework.
+
+It is built via the TSMP2 repository:
+https://github.com/HPSCTerrSys/TSMP2
+
+
 # ParFlow
 
 ![ParFlow Linux CI Test](https://github.com/parflow/parflow/actions/workflows/linux.yml/badge.svg)

cmake/modules/FindOASIS.cmake

@@ -22,5 +22,6 @@ if(OASIS_FOUND AND NOT TARGET OASIS3MCT::OASIS3MCT)
   target_link_libraries(OASIS3MCT::OASIS3MCT INTERFACE ${OASIS_Fortran_LIBRARY} ${MCT_Fortran_LIBRARY} ${MPEU_Fortran_LIBRARY} ${SCRIP_Fortran_LIBRARY})
   target_link_libraries(OASIS3MCT::OASIS3MCT INTERFACE OpenMP::OpenMP_Fortran)
   target_link_libraries(OASIS3MCT::OASIS3MCT INTERFACE ${NETCDF_Fortran_LIBRARY})
+  target_include_directories(OASIS3MCT::OASIS3MCT INTERFACE ${NETCDF_Fortran_INCLUDES})
 endif()
 

pfsimulator/amps/CMakeLists.txt

@@ -23,6 +23,10 @@ set(COMMON_SRC_FILES
   amps_print.c
 )
 
+list(APPEND HEADER_FILES common/amps_common.h
+                        ${PARFLOW_AMPS_LAYER}/amps.h
+                        ${PARFLOW_AMPS_LAYER}/amps_proto.h)
+
 # Add RMM / Umpire wrappers if enabled
 if (${PARFLOW_HAVE_RMM})
   list(APPEND COMMON_SRC_FILES
@@ -111,7 +115,8 @@ endif()
 
 if(${PARFLOW_HAVE_OAS3})
   target_link_libraries(amps PUBLIC OASIS3MCT::OASIS3MCT)
-endif()
+  list(APPEND HEADER_FILES oas3/oas3_coupler.h oas3/oas3_external.h)
+endif(${PARFLOW_HAVE_OAS3})
 
 # -----------------------------
 # Install targets and scripts
@@ -123,5 +128,9 @@ set(AMPS_SCRIPTS run)
 string(REGEX REPLACE "([^;]+)" "${PARFLOW_AMPS_LAYER}/\\1" AMPS_SCRIPTS "${AMPS_SCRIPTS}")
 install(FILES ${AMPS_SCRIPTS} DESTINATION bin)
 
+# Install headers
+install(FILES ${HEADER_FILES}
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/parflow)
+
 # Add subdirectory for tests
 add_subdirectory(test/src)

pfsimulator/amps/mpi1/amps_finalize.c

@@ -68,8 +68,7 @@ int amps_Finalize()
     MPI_Comm_free(&amps_CommNode);
     MPI_Comm_free(&amps_CommWrite);
     MPI_Comm_free(&amps_CommWorld);
-
-    MPI_Finalize();
+    /* MPI_Finalize(); */
   }
 #if defined(PARFLOW_HAVE_CUDA) || defined(PARFLOW_HAVE_KOKKOS)
   amps_gpu_finalize();

pfsimulator/amps/mpi1/amps_init.c

@@ -35,6 +35,8 @@
 #include <unistd.h>
 #include <inttypes.h>
 
+MPI_Comm dacomm; /* kuw */
+
 /* Global flag indicating if AMPS has been initialized */
 int amps_mpi_initialized = FALSE;
 
@@ -114,10 +116,10 @@ int amps_Init(int *argc, char **argv[])
   int length;
 #endif
 
-  MPI_Init(argc, argv);
+  /* MPI_Init(argc, argv); */
   amps_mpi_initialized = TRUE;
 
-  MPI_Comm_dup(MPI_COMM_WORLD, &amps_CommWorld);
+  MPI_Comm_dup(dacomm, &amps_CommWorld);
 
   MPI_Comm_size(amps_CommWorld, &amps_size);
   MPI_Comm_rank(amps_CommWorld, &amps_rank);

pfsimulator/parflow_lib/CMakeLists.txt

@@ -323,3 +323,69 @@ endif (${PARFLOW_HAVE_PDI})
 if (${PARFLOW_HAVE_NETCDF})
   target_link_libraries (pfsimulator NetCDF::NetCDF)
 endif (${PARFLOW_HAVE_NETCDF})
+
+# Install headers
+set (HEADER_FILES
+  backend_mapping.h
+  background.h
+  bc_pressure.h
+  char_vector.h
+  clustering.h
+  communication.h
+  computation.h
+  drv_gridmodule.h
+  drv_module.h
+  drv_tilemodule.h
+  f2c.h
+  fgetopt.h
+  file_versions.h
+  general.h
+  geometry.h
+  geostats.h
+  globals.h
+  grgeometry.h
+  grgeom_list.h
+  grgeom_octree.h
+  grid.h
+  hbt.h
+  hypre_dependences.h
+  index_space.h
+  info_header.h
+  input_database.h
+  kinsol_dependences.h
+  lb.h
+  llnlmath.h
+  llnltyps.h
+  logging.h
+  loops.h
+  matrix.h
+  metadata.h
+  nl_function_eval.h
+  n_vector.h
+  Parflow.hxx
+  parflow.h
+  parflow_netcdf.h
+  parflow_proto_f90.h
+  parflow_proto_f.h
+  parflow_proto.h
+  pf_cudaloops.h
+  pf_cudamalloc.h
+  pf_devices.h
+  pf_hypre.h
+  pf_kokkosloops.h
+  pf_kokkosmalloc.h
+  pf_module.h
+  pf_omploops.h
+  problem_bc.h
+  problem_eval.h
+  problem.h
+  region.h
+  solver.h
+  time_cycle_data.h
+  timing.h
+  vector.h
+  well.h
+  reservoir.h
+)
+install(FILES ${HEADER_FILES}
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/parflow)

pfsimulator/parflow_lib/parflow_proto.h

@@ -978,6 +978,8 @@ void PhaseRelPermFreeInstanceXtra(void);
 PFModule *PhaseRelPermNewPublicXtra(void);
 void PhaseRelPermFreePublicXtra(void);
 int PhaseRelPermSizeOfTempData(void);
+Vector *PhaseRelPermGetAlpha(PFModule *this_module);
+Vector *PhaseRelPermGetN(PFModule *this_module);
 
 typedef void (*PhaseSourceInvoke) (Vector *phase_source, int phase, Problem *problem, ProblemData *problem_data, double time);
 typedef PFModule *(*PhaseSourceNewPublicXtraInvoke) (int num_phases);
@@ -1031,6 +1033,8 @@ typedef PFModule *(*SaturationOutputStaticInvoke) (char *file_prefix, ProblemDat
 
 /* problem_saturation.c */
 void Saturation(Vector *phase_saturation, Vector *phase_pressure, Vector *phase_density, double gravity, ProblemData *problem_data, int fcn);
+Vector *SaturationGetAlpha(PFModule *this_module);
+Vector *SaturationGetN(PFModule *this_module);
 PFModule *SaturationInitInstanceXtra(Grid *grid, double *temp_data);
 void SaturationFreeInstanceXtra(void);
 PFModule *SaturationNewPublicXtra(void);
@@ -1238,6 +1242,8 @@ PFModule *SolverRichardsNewPublicXtra(char *name);
 void SolverRichardsFreePublicXtra(void);
 int SolverRichardsSizeOfTempData(void);
 ProblemData *GetProblemDataRichards(PFModule *this_module);
+PFModule *GetPhaseRelPerm(PFModule *this_module);
+PFModule *GetSaturation(PFModule *this_module);
 Problem  *GetProblemRichards(PFModule *this_module);
 PFModule *GetICPhasePressureRichards(PFModule *this_module);
 Grid *GetGrid2DRichards(PFModule *this_module);
@@ -1251,6 +1257,16 @@ void AdvanceRichards(PFModule *this_module,
                      Vector ** porosity_out,
                      Vector ** saturation_out
                      );
+// PDAF: this function is for initialization of OASIS only
+void PseudoAdvanceRichards(PFModule *this_module,
+                     double    start_time,   /* Starting time */
+                     double    stop_time,    /* Stopping time */
+                     PFModule *time_step_control, /* Use this module to control timestep if supplied */
+                     Vector *  evap_trans,   /* Flux from land surface model */
+                     Vector ** pressure_out, /* Output vars */
+                     Vector ** porosity_out,
+                     Vector ** saturation_out
+                     );
 void ExportRichards(PFModule *this_module,
                     Vector ** pressure_out,  /* Output vars */
                     Vector ** porosity_out,

pfsimulator/parflow_lib/problem_phase_rel_perm.c

@@ -2179,3 +2179,31 @@ int  PhaseRelPermSizeOfTempData()
 
   return sz;
 }
+
+/*-----------------------------------------------------------------------
+ * PhaseRelPermGetAlpha
+ *-----------------------------------------------------------------------*/
+
+Vector *PhaseRelPermGetAlpha(PFModule *this_module)
+{
+  PublicXtra 	*public_xtra = (PublicXtra*)PFModulePublicXtra(this_module);
+
+  Type1 	*dummy1;
+  dummy1 = (Type1*)(public_xtra->data);
+
+  return (dummy1 -> alpha_values);
+}
+
+/*-----------------------------------------------------------------------
+ * PhaseRelPermGetN
+ *-----------------------------------------------------------------------*/
+
+Vector *PhaseRelPermGetN(PFModule *this_module)
+{
+  PublicXtra 	*public_xtra = (PublicXtra*)PFModulePublicXtra(this_module);
+
+  Type1 	*dummy1;
+  dummy1 = (Type1*)(public_xtra->data);
+
+  return (dummy1 -> n_values);
+}

pfsimulator/parflow_lib/problem_saturation.c

@@ -654,6 +654,34 @@ void     Saturation(
   }          /* End switch */
 }
 
+/*--------------------------------------------------------------------------
+ * SaturationGetAlpha
+ *--------------------------------------------------------------------------*/
+
+Vector *SaturationGetAlpha(PFModule *this_module)
+{
+    PublicXtra *public_xtra = (PublicXtra *)PFModulePublicXtra(this_module);
+
+    Type1 *dummy1;
+    dummy1 = (Type1 *)(public_xtra -> data);
+
+    return (dummy1 -> alpha_values);
+}
+
+/*--------------------------------------------------------------------------
+ * SaturationGetN
+ *--------------------------------------------------------------------------*/
+
+Vector *SaturationGetN(PFModule *this_module)
+{
+    PublicXtra *public_xtra = (PublicXtra *)PFModulePublicXtra(this_module);
+
+    Type1 *dummy1;
+    dummy1 = (Type1 *)(public_xtra -> data);
+
+    return (dummy1 -> n_values);
+}
+
 /*--------------------------------------------------------------------------
  * SaturationInitInstanceXtra
  *--------------------------------------------------------------------------*/