Remote link discovery (#1572)

* Changed restart so that is can restart with new parallelism as long as the total number of threads in the run is the same (i.e. no need to change what is in each partition, just remap partitions to new rank/thread pairs). For any new parallelism, SST will automatically discover which rank remote Links are on."

* Fixed link connection logic in test_StatisticsComponent_basic.py and add tests that do remapping on a restart.

* Move some Link APIs to be protected and add functions to ThreadSync which is a friend of Link so child classes can access the moved functionality.

* Fix typos in comments and output strings

Author: feldergast

src/sst/core/link.cc

@@ -33,6 +33,8 @@
 
 namespace SST {
 
+bool Link::is_restart_same_parallelism = false;
+
 void
 SST::Core::Serialization::serialize_impl<Link*>::serialize_events(
     serializer& ser, uintptr_t delivery_info, ActivityQueue* queue)
@@ -270,8 +272,9 @@ SST::Core::Serialization::serialize_impl<Link*>::operator()(Link*& s, serializer
             if ( pair_restart_rank.rank == RankInfo::UNASSIGNED ) pair_restart_rank = pair_rank;
         }
 
-        bool is_restart_sync = (my_restart_rank != pair_restart_rank);
-
+        // Need to know if this link is a sync link or not.  We can only do this if the restart uses the exact
+        // rank/thread counts as the checkpoint.  This info is stored in Link::is_restart_same_parallelism
+        bool is_restart_sync = (Link::is_restart_same_parallelism && my_restart_rank != pair_restart_rank);
 
         /*
           Create or get link from tracker

src/sst/core/link.h

@@ -316,27 +316,6 @@ class alignas(64) Link
     */
     bool isConfigured() { return type != UNINITIALIZED; }
 
-    /**
-       Get the latency on the link in units of core atomic time base
-
-       NOTE: This is a core only API and not part of the public stable API
-    */
-    SimTime_t getLatency() { return latency; }
-
-    /**
-       Get the delivery_info for the link
-
-       NOTE: This is a core only API and not part of the public stable API
-    */
-    uintptr_t getDeliveryInfo() { return delivery_info; }
-
-    /**
-       Get the pair_link
-
-       NOTE: This is a core only API and not part of the public stable API
-    */
-    Link* getPairLink() { return pair_link; }
-
 
 #ifdef __SST_DEBUG_EVENT_TRACKING__
     void setSendingComponentInfo(const std::string& comp_in, const std::string& type_in, const std::string& port_in)
@@ -379,6 +358,23 @@ class alignas(64) Link
         // else pair_link->tag = 0x80000000 | new_tag;
     }
 
+    /**
+       Get the latency on the link in units of core atomic time base
+
+       NOTE: This is a core only API and not part of the public stable API
+    */
+    SimTime_t getLatency() { return latency; }
+
+    /**
+       Get the delivery_info for the link
+    */
+    uintptr_t getDeliveryInfo() { return delivery_info; }
+
+    /**
+       Get the pair_link
+    */
+    Link* getPairLink() { return pair_link; }
+
     /**
        Sends an Event over a Link with an additional delay specified with a TimeConverter. I.e. the total delay is the
        Link's delay + the additional specified delay.
@@ -402,6 +398,12 @@ class alignas(64) Link
         event->updateDeliveryInfo(delivery_info);
     }
 
+    /**
+       Variable used by Link restarts to know whether stored rank data for remote links is still valid (i.e. did we
+       restart with the exact same rank/thread count or not).
+    */
+    static bool is_restart_same_parallelism;
+
     // Since Links are found in pairs, I will keep all the information
     // needed for me to send and deliver an event to the other side of
     // the Link.  That means, that I mostly keep my pair's

src/sst/core/main.cc

@@ -518,7 +518,7 @@ start_simulation(uint32_t tid, SimThreadInfo_t& info, Core::ThreadSafe::Barrier&
     if ( restart ) {
 
         // Finish parsing checkpoint for restart
-        sim->restart();
+        sim->restart(info.graph);
 
         barrier.wait();
 
@@ -528,6 +528,17 @@ start_simulation(uint32_t tid, SimThreadInfo_t& info, Core::ThreadSafe::Barrier&
 
         barrier.wait();
 
+        // Need to detect the sync intervals
+        if ( info.myRank.thread == 0 ) {
+            sim->findRankSyncInterval();
+        }
+
+        barrier.wait();
+
+        sim->findThreadSyncInterval();
+
+        barrier.wait();
+
     } // if ( restart )
 
 

src/sst/core/model/configGraph.cc

@@ -627,12 +627,13 @@ ConfigGraph::splitGraph(const std::set<uint32_t>& orig_rank_set, const std::set<
         graph = new ConfigGraph();
 
         // Need to copy over any restart data
-        graph->cpt_ranks           = cpt_ranks;
-        graph->cpt_currentSimCycle = cpt_currentSimCycle;
-        graph->cpt_currentPriority = cpt_currentPriority;
-        graph->cpt_minPart         = cpt_minPart;
-        graph->cpt_minPartTC       = cpt_minPartTC;
-        graph->cpt_max_event_id    = cpt_max_event_id;
+        graph->cpt_ranks            = cpt_ranks;
+        graph->cpt_currentSimCycle  = cpt_currentSimCycle;
+        graph->cpt_currentPriority  = cpt_currentPriority;
+        graph->cpt_minPart          = cpt_minPart;
+        graph->cpt_minPartTC        = cpt_minPartTC;
+        graph->cpt_max_event_id     = cpt_max_event_id;
+        graph->cpt_remap_partitions = cpt_remap_partitions;
 
         graph->cpt_libnames       = cpt_libnames;
         graph->cpt_shared_objects = cpt_shared_objects;

src/sst/core/model/configGraph.h

@@ -211,6 +211,7 @@ class ConfigGraph : public SST::Core::Serialization::serializable
         SST_SER(cpt_minPart);
         SST_SER(cpt_minPartTC);
         SST_SER(cpt_max_event_id);
+        SST_SER(cpt_remap_partitions);
 
         SST_SER(*(cpt_libnames.get()));
         SST_SER(*(cpt_shared_objects.get()));
@@ -225,7 +226,8 @@ class ConfigGraph : public SST::Core::Serialization::serializable
     int           cpt_currentPriority = 0;
     SimTime_t     cpt_minPart         = std::numeric_limits<SimTime_t>::max();
     TimeConverter cpt_minPartTC;
-    uint64_t      cpt_max_event_id = 0;
+    uint64_t      cpt_max_event_id     = 0;
+    bool          cpt_remap_partitions = false;
 
     std::shared_ptr<std::set<std::string>> cpt_libnames       = std::make_shared<std::set<std::string>>();
     std::shared_ptr<std::vector<char>>     cpt_shared_objects = std::make_shared<std::vector<char>>();

src/sst/core/model/restart/sstcptmodel.cc

@@ -121,11 +121,17 @@ SSTCPTModelDefinition::createConfigGraph()
     if ( (cfg.num_ranks() != graph->cpt_ranks.rank || cfg.num_threads() != graph->cpt_ranks.thread) &&
          !(cfg.num_threads() == 1 && cfg.num_ranks() == 1) ) {
 
-        Output::getDefaultObject().fatal(CALL_INFO, 1,
-            "Rank or thread counts do not match checkpoint. "
-            "Checkpoint requires %" PRIu32 " ranks and %" PRIu32 " threads. "
-            "Serial restarts are also permitted.\n",
-            graph->cpt_ranks.rank, graph->cpt_ranks.thread);
+        // We can proceed if the total number of partitions is the same, otherwise, error
+        if ( (cfg.num_ranks() * cfg.num_threads()) == (graph->cpt_ranks.rank * graph->cpt_ranks.thread) ) {
+            graph->cpt_remap_partitions = true;
+        }
+        else {
+            Output::getDefaultObject().fatal(CALL_INFO, 1,
+                "Rank or thread counts do not match checkpoint. Checkpoint/restart requires that the total parallelism "
+                "be the same between a checkpoint and restart (i.e. ranks * threads is the same for both).  Checkpoint "
+                "was created with %" PRIu32 " ranks and %" PRIu32 " threads. Serial restarts are also permitted.\n",
+                graph->cpt_ranks.rank, graph->cpt_ranks.thread);
+        }
     }
     /******** ^^ Works for regular and N->1 restart ^^ ***********/