1 /**
  2  * Sets up a replica set. To make the set running, call {@link #startSet},
  3  * followed by {@link #initiate} (and optionally,
  4  * {@link #awaitSecondaryNodes} to block till the  set is fully operational).
  5  * Note that some of the replica start up parameters are not passed here,
  6  * but to the #startSet method.
  7  *
  8  * @param {Object|string} opts If this value is a string, it specifies the connection string for
  9  *      a MongoD host to be used for recreating a ReplSetTest from. Otherwise, if it is an object,
 10  *      it must have the following contents:
 11  *
 12  *   {
 13  *     name {string}: name of this replica set. Default: 'testReplSet'
 14  *     host {string}: name of the host machine. Hostname will be used
 15  *        if not specified.
 16  *     useHostName {boolean}: if true, use hostname of machine,
 17  *        otherwise use localhost
 18  *     nodes {number|Object|Array.<Object>}: number of replicas. Default: 0.
 19  *        Can also be an Object (or Array).
 20  *        Format for Object:
 21  *          {
 22  *            <any string>: replica member option Object. @see MongoRunner.runMongod
 23  *            <any string2>: and so on...
 24  *          }
 25  *          If object has a special "rsConfig" field then those options will be used for each
 26  *          replica set member config options when used to initialize the replica set, or
 27  *          building the config with getReplSetConfig()
 28  *
 29  *        Format for Array:
 30  *           An array of replica member option Object. @see MongoRunner.runMongod
 31  *
 32  *        Note: For both formats, a special boolean property 'arbiter' can be
 33  *          specified to denote a member is an arbiter.
 34  *
 35  *        Note: A special "bridgeOptions" property can be specified in both the object and array
 36  *           formats to configure the options for the mongobridge corresponding to that node. These
 37  *           options are merged with the opts.bridgeOptions options, where the node-specific options
 38  *           take precedence.
 39  *
 40  *     nodeOptions {Object}: Options to apply to all nodes in the replica set.
 41  *        Format for Object:
 42  *          { cmdline-param-with-no-arg : "",
 43  *            param-with-arg : arg }
 44  *        This turns into "mongod --cmdline-param-with-no-arg --param-with-arg arg"
 45  *
 46  *     oplogSize {number}: Default: 40
 47  *     useSeedList {boolean}: Use the connection string format of this set
 48  *        as the replica set name (overrides the name property). Default: false
 49  *     keyFile {string}
 50  *     protocolVersion {number}: protocol version of replset used by the replset initiation.
 51  *
 52  *     useBridge {boolean}: If true, then a mongobridge process is started for each node in the
 53  *        replica set. Both the replica set configuration and the connections returned by startSet()
 54  *        will be references to the proxied connections. Defaults to false.
 55  *
 56  *     bridgeOptions {Object}: Options to apply to all mongobridge processes. Defaults to {}.
 57  *
 58  *     settings {object}: Setting used in the replica set config document.
 59  *        Example:
 60  *              settings: { chainingAllowed: false, ... }
 61  *   }
 62  *
 63  * Member variables:
 64  *  nodes {Array.<Mongo>} - connection to replica set members
 65  */
 66 var ReplSetTest = function(opts) {
 67     'use strict';
 68 
 69     if (!(this instanceof ReplSetTest)) {
 70         return new ReplSetTest(opts);
 71     }
 72 
 73     // Capture the 'this' reference
 74     var self = this;
 75 
 76     // Replica set health states
 77     var Health = {UP: 1, DOWN: 0};
 78 
 79     var _alldbpaths;
 80     var _configSettings;
 81 
 82     // mongobridge related variables. Only available if the bridge option is selected.
 83     var _useBridge;
 84     var _bridgeOptions;
 85     var _unbridgedPorts;
 86     var _unbridgedNodes;
 87 
 88     // Publicly exposed variables
 89 
 90     /**
 91      * Populates a reference to all reachable nodes.
 92      */
 93     function _clearLiveNodes() {
 94         self.liveNodes = {master: null, slaves: []};
 95     }
 96 
 97     /**
 98      * Returns the config document reported from the specified connection.
 99      */
100     function _replSetGetConfig(conn) {
101         return assert.commandWorked(conn.adminCommand({replSetGetConfig: 1})).config;
102     }
103 
104     /**
105      * Invokes the 'ismaster' command on each individual node and returns whether the node is the
106      * current RS master.
107      */
108     function _callIsMaster() {
109         _clearLiveNodes();
110 
111         var twoPrimaries = false;
112         self.nodes.forEach(function(node) {
113             try {
114                 var n = node.getDB('admin').runCommand({ismaster: 1});
115                 if (n.ismaster == true) {
116                     if (self.liveNodes.master) {
117                         twoPrimaries = true;
118                     } else {
119                         self.liveNodes.master = node;
120                     }
121                 } else {
122                     node.setSlaveOk();
123                     self.liveNodes.slaves.push(node);
124                 }
125             } catch (err) {
126                 print("ReplSetTest Could not call ismaster on node " + node + ": " + tojson(err));
127             }
128         });
129         if (twoPrimaries) {
130             return false;
131         }
132 
133         return self.liveNodes.master || false;
134     }
135 
136     /**
137      * Wait for a rs indicator to go to a particular state or states.
138      *
139      * @param node is a single node or list of nodes, by id or conn
140      * @param states is a single state or list of states
141      * @param ind is the indicator specified
142      * @param timeout how long to wait for the state to be reached
143      */
144     function _waitForIndicator(node, states, ind, timeout) {
145         if (node.length) {
146             var nodes = node;
147             for (var i = 0; i < nodes.length; i++) {
148                 if (states.length)
149                     _waitForIndicator(nodes[i], states[i], ind, timeout);
150                 else
151                     _waitForIndicator(nodes[i], states, ind, timeout);
152             }
153 
154             return;
155         }
156 
157         timeout = timeout || 30000;
158 
159         if (!node.getDB) {
160             node = self.nodes[node];
161         }
162 
163         if (!states.length) {
164             states = [states];
165         }
166 
167         print("ReplSetTest waitForIndicator " + ind + " on " + node);
168         printjson(states);
169         print("ReplSetTest waitForIndicator from node " + node);
170 
171         var lastTime = null;
172         var currTime = new Date().getTime();
173         var status;
174 
175         assert.soon(function() {
176             try {
177                 var conn = _callIsMaster();
178                 if (!conn) {
179                     conn = self.liveNodes.slaves[0];
180                 }
181 
182                 // Try again to load connection
183                 if (!conn)
184                     return false;
185 
186                 var getStatusFunc = function() {
187                     status = conn.getDB('admin').runCommand({replSetGetStatus: 1});
188                 };
189 
190                 if (self.keyFile) {
191                     // Authenticate connection used for running replSetGetStatus if needed
192                     authutil.asCluster(conn, self.keyFile, getStatusFunc);
193                 } else {
194                     getStatusFunc();
195                 }
196             } catch (ex) {
197                 print("ReplSetTest waitForIndicator could not get status: " + tojson(ex));
198                 return false;
199             }
200 
201             var printStatus = false;
202             if (lastTime == null || (currTime = new Date().getTime()) - (1000 * 5) > lastTime) {
203                 if (lastTime == null) {
204                     print("ReplSetTest waitForIndicator Initial status (timeout : " + timeout +
205                           ") :");
206                 }
207 
208                 printjson(status);
209                 lastTime = new Date().getTime();
210                 printStatus = true;
211             }
212 
213             if (typeof status.members == 'undefined') {
214                 return false;
215             }
216 
217             for (var i = 0; i < status.members.length; i++) {
218                 if (printStatus) {
219                     print("Status for : " + status.members[i].name + ", checking " + node.host +
220                           "/" + node.name);
221                 }
222 
223                 if (status.members[i].name == node.host || status.members[i].name == node.name) {
224                     for (var j = 0; j < states.length; j++) {
225                         if (printStatus) {
226                             print("Status -- " + " current state: " + status.members[i][ind] +
227                                   ",  target state : " + states[j]);
228                         }
229 
230                         if (typeof(states[j]) != "number") {
231                             throw new Error("State was not an number -- type:" + typeof(states[j]) +
232                                             ", value:" + states[j]);
233                         }
234                         if (status.members[i][ind] == states[j]) {
235                             return true;
236                         }
237                     }
238                 }
239             }
240 
241             return false;
242 
243         }, "waiting for state indicator " + ind + " for " + timeout + "ms", timeout);
244 
245         print("ReplSetTest waitForIndicator final status:");
246         printjson(status);
247     }
248 
249     /**
250      * Wait for a health indicator to go to a particular state or states.
251      *
252      * @param node is a single node or list of nodes, by id or conn
253      * @param state is a single state or list of states. ReplSetTest.Health.DOWN can
254      *     only be used in cases when there is a primary available or slave[0] can
255      *     respond to the isMaster command.
256      */
257     function _waitForHealth(node, state, timeout) {
258         _waitForIndicator(node, state, "health", timeout);
259     }
260 
261     /**
262      * Returns the optime for the specified host by issuing replSetGetStatus.
263      */
264     function _getLastOpTime(conn) {
265         var replSetStatus =
266             assert.commandWorked(conn.getDB("admin").runCommand({replSetGetStatus: 1}));
267         var connStatus = replSetStatus.members.filter(m => m.self)[0];
268         return connStatus.optime;
269     }
270 
271     /**
272      * Returns the OpTime timestamp for the specified host by issuing replSetGetStatus.
273      */
274     function _getLastOpTimeTimestamp(conn) {
275         var myOpTime = _getLastOpTime(conn);
276         if (!myOpTime) {
277             // Must be an ARBITER
278             return undefined;
279         }
280 
281         return myOpTime.ts ? myOpTime.ts : myOpTime;
282     }
283 
284     /**
285      * Returns the last committed OpTime for the replicaset as known by the host.
286      * This function may return an OpTime with Timestamp(0,0) and Term(0) if there is no
287      * last committed OpTime.
288      */
289     function _getLastCommittedOpTime(conn) {
290         var replSetStatus =
291             assert.commandWorked(conn.getDB("admin").runCommand({replSetGetStatus: 1}));
292         return (replSetStatus.OpTimes || replSetStatus.optimes).lastCommittedOpTime ||
293             {ts: Timestamp(0, 0), t: NumberLong(0)};
294     }
295 
296     /**
297      * Returns the {readConcern: majority} OpTime for the host.
298      * This is the OpTime of the host's "majority committed" snapshot.
299      * This function may return an OpTime with Timestamp(0,0) and Term(0) if read concern majority
300      * is not enabled, or if there has not been a committed snapshot yet.
301      */
302     function _getReadConcernMajorityOpTime(conn) {
303         var replSetStatus =
304             assert.commandWorked(conn.getDB("admin").runCommand({replSetGetStatus: 1}));
305         return (replSetStatus.OpTimes || replSetStatus.optimes).readConcernMajorityOpTime ||
306             {ts: Timestamp(0, 0), t: NumberLong(0)};
307     }
308 
309     /**
310      * Returns the last durable OpTime timestamp for the host if running with journaling.
311      * Returns the last applied OpTime timestamp otherwise.
312      */
313     function _getDurableOpTimeTimestamp(conn) {
314         var replSetStatus =
315             assert.commandWorked(conn.getDB("admin").runCommand({replSetGetStatus: 1}));
316 
317         var runningWithoutJournaling = TestData.noJournal || "inMemory" == TestData.storageEngine ||
318             "ephemeralForTest" == TestData.storageEngine;
319         var opTimeType = "durableOpTime";
320         if (runningWithoutJournaling) {
321             opTimeType = "appliedOpTime";
322         }
323         return replSetStatus.optimes[opTimeType].ts;
324     }
325 
326     function _isEarlierTimestamp(ts1, ts2) {
327         if (ts1.getTime() == ts2.getTime()) {
328             return ts1.getInc() < ts2.getInc();
329         }
330         return ts1.getTime() < ts2.getTime();
331     }
332 
333     function _isEarlierOpTime(ot1, ot2) {
334         // Make sure both optimes have a timestamp and a term.
335         ot1 = ot1.t ? ot1 : {ts: ot1, t: NumberLong(-1)};
336         ot2 = ot2.t ? ot2 : {ts: ot2, t: NumberLong(-1)};
337 
338         // If both optimes have a term that's not -1 and one has a lower term, return that optime.
339         if (!friendlyEqual(ot1.t, NumberLong(-1)) && !friendlyEqual(ot2.t, NumberLong(-1))) {
340             if (!friendlyEqual(ot1.t, ot2.t)) {
341                 return ot1.t < ot2.t;
342             }
343         }
344 
345         // Otherwise, choose the optime with the lower timestamp.
346         return _isEarlierTimestamp(ot1.ts, ot2.ts);
347     }
348 
349     /**
350      * Returns list of nodes as host:port strings.
351      */
352     this.nodeList = function() {
353         var list = [];
354         for (var i = 0; i < this.ports.length; i++) {
355             list.push(this.host + ":" + this.ports[i]);
356         }
357 
358         return list;
359     };
360 
361     this.getNodeId = function(node) {
362         if (node.toFixed) {
363             return parseInt(node);
364         }
365 
366         for (var i = 0; i < this.nodes.length; i++) {
367             if (this.nodes[i] == node) {
368                 return i;
369             }
370         }
371 
372         if (node instanceof ObjectId) {
373             for (i = 0; i < this.nodes.length; i++) {
374                 if (this.nodes[i].runId == node) {
375                     return i;
376                 }
377             }
378         }
379 
380         if (node.nodeId != null) {
381             return parseInt(node.nodeId);
382         }
383 
384         return undefined;
385     };
386 
387     this.getPort = function(n) {
388         var n = this.getNodeId(n);
389         return this.ports[n];
390     };
391 
392     this._addPath = function(p) {
393         if (!_alldbpaths)
394             _alldbpaths = [p];
395         else
396             _alldbpaths.push(p);
397 
398         return p;
399     };
400 
401     this.getReplSetConfig = function() {
402         var cfg = {};
403         cfg._id = this.name;
404 
405         if (this.protocolVersion !== undefined && this.protocolVersion !== null) {
406             cfg.protocolVersion = this.protocolVersion;
407         }
408 
409         cfg.members = [];
410 
411         for (var i = 0; i < this.ports.length; i++) {
412             var member = {};
413             member._id = i;
414 
415             var port = this.ports[i];
416             member.host = this.host + ":" + port;
417 
418             var nodeOpts = this.nodeOptions["n" + i];
419             if (nodeOpts) {
420                 if (nodeOpts.arbiter) {
421                     member.arbiterOnly = true;
422                 }
423 
424                 if (nodeOpts.rsConfig) {
425                     Object.extend(member, nodeOpts.rsConfig);
426                 }
427             }
428 
429             cfg.members.push(member);
430         }
431 
432         if (jsTestOptions().useLegacyReplicationProtocol) {
433             cfg.protocolVersion = 0;
434         }
435 
436         if (_configSettings) {
437             cfg.settings = _configSettings;
438         }
439 
440         return cfg;
441     };
442 
443     this.getURL = function() {
444         var hosts = [];
445 
446         for (var i = 0; i < this.ports.length; i++) {
447             hosts.push(this.host + ":" + this.ports[i]);
448         }
449 
450         return this.name + "/" + hosts.join(",");
451     };
452 
453     /**
454      * Starts each node in the replica set with the given options.
455      *
456      * @param options - The options passed to {@link MongoRunner.runMongod}
457      */
458     this.startSet = function(options) {
459         print("ReplSetTest starting set");
460 
461         var nodes = [];
462         for (var n = 0; n < this.ports.length; n++) {
463             nodes.push(this.start(n, options));
464         }
465 
466         this.nodes = nodes;
467         return this.nodes;
468     };
469 
470     /**
471      * Blocks until the secondary nodes have completed recovery and their roles are known.
472      */
473     this.awaitSecondaryNodes = function(timeout) {
474         timeout = timeout || 60000;
475 
476         assert.soonNoExcept(function() {
477             // Reload who the current slaves are
478             self.getPrimary(timeout);
479 
480             var slaves = self.liveNodes.slaves;
481             var len = slaves.length;
482             var ready = true;
483 
484             for (var i = 0; i < len; i++) {
485                 var isMaster = slaves[i].getDB("admin").runCommand({ismaster: 1});
486                 var arbiter = (isMaster.arbiterOnly == undefined ? false : isMaster.arbiterOnly);
487                 ready = ready && (isMaster.secondary || arbiter);
488             }
489 
490             return ready;
491         }, "Awaiting secondaries", timeout);
492     };
493 
494     /**
495      * Blocks until all nodes agree on who the primary is.
496      */
497     this.awaitNodesAgreeOnPrimary = function(timeout) {
498         timeout = timeout || 60000;
499 
500         assert.soonNoExcept(function() {
501             var primary = -1;
502 
503             for (var i = 0; i < self.nodes.length; i++) {
504                 var replSetGetStatus =
505                     self.nodes[i].getDB("admin").runCommand({replSetGetStatus: 1});
506                 var nodesPrimary = -1;
507                 for (var j = 0; j < replSetGetStatus.members.length; j++) {
508                     if (replSetGetStatus.members[j].state === ReplSetTest.State.PRIMARY) {
509                         // Node sees two primaries.
510                         if (nodesPrimary !== -1) {
511                             return false;
512                         }
513                         nodesPrimary = j;
514                     }
515                 }
516                 // Node doesn't see a primary.
517                 if (nodesPrimary < 0) {
518                     return false;
519                 }
520 
521                 if (primary < 0) {
522                     // If we haven't seen a primary yet, set it to this.
523                     primary = nodesPrimary;
524                 } else if (primary !== nodesPrimary) {
525                     return false;
526                 }
527             }
528 
529             return true;
530         }, "Awaiting nodes to agree on primary", timeout);
531     };
532 
533     /**
534      * Blocking call, which will wait for a primary to be elected for some pre-defined timeout and
535      * if primary is available will return a connection to it. Otherwise throws an exception.
536      */
537     this.getPrimary = function(timeout) {
538         timeout = timeout || 60000;
539         var primary = null;
540 
541         assert.soonNoExcept(function() {
542             primary = _callIsMaster();
543             return primary;
544         }, "Finding primary", timeout);
545 
546         return primary;
547     };
548 
549     this.awaitNoPrimary = function(msg, timeout) {
550         msg = msg || "Timed out waiting for there to be no primary in replset: " + this.name;
551         timeout = timeout || 30000;
552 
553         assert.soonNoExcept(function() {
554             return _callIsMaster() == false;
555         }, msg, timeout);
556     };
557 
558     this.getSecondaries = function(timeout) {
559         var master = this.getPrimary(timeout);
560         var secs = [];
561         for (var i = 0; i < this.nodes.length; i++) {
562             if (this.nodes[i] != master) {
563                 secs.push(this.nodes[i]);
564             }
565         }
566 
567         return secs;
568     };
569 
570     this.getSecondary = function(timeout) {
571         return this.getSecondaries(timeout)[0];
572     };
573 
574     this.status = function(timeout) {
575         var master = _callIsMaster();
576         if (!master) {
577             master = this.liveNodes.slaves[0];
578         }
579 
580         return master.getDB("admin").runCommand({replSetGetStatus: 1});
581     };
582 
583     /**
584      * Adds a node to the replica set managed by this instance.
585      */
586     this.add = function(config) {
587         var nextPort = allocatePort();
588         print("ReplSetTest Next port: " + nextPort);
589 
590         this.ports.push(nextPort);
591         printjson(this.ports);
592 
593         if (_useBridge) {
594             _unbridgedPorts.push(allocatePort());
595         }
596 
597         var nextId = this.nodes.length;
598         printjson(this.nodes);
599 
600         print("ReplSetTest nextId: " + nextId);
601         return this.start(nextId, config);
602     };
603 
604     this.remove = function(nodeId) {
605         nodeId = this.getNodeId(nodeId);
606         this.nodes.splice(nodeId, 1);
607         this.ports.splice(nodeId, 1);
608 
609         if (_useBridge) {
610             _unbridgedPorts.splice(nodeId, 1);
611             _unbridgedNodes.splice(nodeId, 1);
612         }
613     };
614 
615     this._setDefaultConfigOptions = function(config) {
616         if (jsTestOptions().useLegacyReplicationProtocol &&
617             !config.hasOwnProperty("protocolVersion")) {
618             config.protocolVersion = 0;
619         }
620     };
621 
622     this.initiate = function(cfg, initCmd, timeout) {
623         var master = this.nodes[0].getDB("admin");
624         var config = cfg || this.getReplSetConfig();
625         var cmd = {};
626         var cmdKey = initCmd || 'replSetInitiate';
627         timeout = timeout || 120000;
628 
629         this._setDefaultConfigOptions(config);
630 
631         cmd[cmdKey] = config;
632         printjson(cmd);
633 
634         assert.commandWorked(master.runCommand(cmd), tojson(cmd));
635         this.awaitSecondaryNodes(timeout);
636 
637         // Setup authentication if running test with authentication
638         if ((jsTestOptions().keyFile) && cmdKey == 'replSetInitiate') {
639             master = this.getPrimary();
640             jsTest.authenticateNodes(this.nodes);
641         }
642     };
643 
644     /**
645      * Gets the current replica set config from the specified node index. If no nodeId is specified,
646      * uses the primary node.
647      */
648     this.getReplSetConfigFromNode = function(nodeId) {
649         if (nodeId == undefined) {
650             // Use 90 seconds timeout for finding a primary
651             return _replSetGetConfig(self.getPrimary(90 * 1000));
652         }
653 
654         if (!isNumber(nodeId)) {
655             throw Error(nodeId + ' is not a number');
656         }
657 
658         return _replSetGetConfig(self.nodes[nodeId]);
659     };
660 
661     this.reInitiate = function() {
662         var config = this.getReplSetConfig();
663         var newVersion = this.getReplSetConfigFromNode().version + 1;
664         config.version = newVersion;
665 
666         this._setDefaultConfigOptions(config);
667 
668         try {
669             assert.commandWorked(this.getPrimary().adminCommand({replSetReconfig: config}));
670         } catch (e) {
671             if (tojson(e).indexOf("error doing query: failed") < 0) {
672                 throw e;
673             }
674         }
675     };
676 
677     /**
678      * Waits for the last oplog entry on the primary to be visible in the committed snapshop view
679      * of the oplog on *all* secondaries.
680      * Returns last oplog entry.
681      */
682     this.awaitLastOpCommitted = function() {
683         var rst = this;
684         var master = rst.getPrimary();
685         var masterOpTime = _getLastOpTime(master);
686 
687         print("Waiting for op with OpTime " + tojson(masterOpTime) +
688               " to be committed on all secondaries");
689 
690         assert.soonNoExcept(function() {
691             for (var i = 0; i < rst.nodes.length; i++) {
692                 var node = rst.nodes[i];
693 
694                 // Continue if we're connected to an arbiter
695                 var res = assert.commandWorked(node.adminCommand({replSetGetStatus: 1}));
696                 if (res.myState == ReplSetTest.State.ARBITER) {
697                     continue;
698                 }
699                 var rcmOpTime = _getReadConcernMajorityOpTime(node);
700                 if (friendlyEqual(rcmOpTime, {ts: Timestamp(0, 0), t: NumberLong(0)})) {
701                     return false;
702                 }
703                 if (_isEarlierOpTime(rcmOpTime, masterOpTime)) {
704                     return false;
705                 }
706             }
707 
708             return true;
709         }, "Op with OpTime " + tojson(masterOpTime) + " failed to be committed on all secondaries");
710 
711         return masterOpTime;
712     };
713 
714     // Wait until the optime of the specified type reaches the primary's last applied optime.
715     this.awaitReplication = function(timeout, secondaryOpTimeType) {
716         timeout = timeout || 30000;
717         secondaryOpTimeType = secondaryOpTimeType || ReplSetTest.OpTimeType.LAST_APPLIED;
718 
719         var masterLatestOpTime;
720 
721         // Blocking call, which will wait for the last optime written on the master to be available
722         var awaitLastOpTimeWrittenFn = function() {
723             var master = self.getPrimary();
724             assert.soonNoExcept(function() {
725                 try {
726                     masterLatestOpTime = _getLastOpTimeTimestamp(master);
727                 } catch (e) {
728                     print("ReplSetTest caught exception " + e);
729                     return false;
730                 }
731 
732                 return true;
733             }, "awaiting oplog query", 30000);
734         };
735 
736         awaitLastOpTimeWrittenFn();
737 
738         // get the latest config version from master. if there is a problem, grab master and try
739         // again
740         var configVersion;
741         var masterOpTime;
742         var masterName;
743         var master;
744 
745         try {
746             master = this.getPrimary();
747             configVersion = this.getReplSetConfigFromNode().version;
748             masterOpTime = _getLastOpTimeTimestamp(master);
749             masterName = master.toString().substr(14);  // strip "connection to "
750         } catch (e) {
751             master = this.getPrimary();
752             configVersion = this.getReplSetConfigFromNode().version;
753             masterOpTime = _getLastOpTimeTimestamp(master);
754             masterName = master.toString().substr(14);  // strip "connection to "
755         }
756 
757         print("ReplSetTest awaitReplication: starting: timestamp for primary, " + masterName +
758               ", is " + tojson(masterLatestOpTime) + ", last oplog entry is " +
759               tojsononeline(masterOpTime));
760 
761         assert.soonNoExcept(function() {
762             try {
763                 print("ReplSetTest awaitReplication: checking secondaries against timestamp " +
764                       tojson(masterLatestOpTime));
765                 var secondaryCount = 0;
766                 for (var i = 0; i < self.liveNodes.slaves.length; i++) {
767                     var slave = self.liveNodes.slaves[i];
768                     var slaveName = slave.toString().substr(14);  // strip "connection to "
769 
770                     var slaveConfigVersion =
771                         slave.getDB("local")['system.replset'].findOne().version;
772 
773                     if (configVersion != slaveConfigVersion) {
774                         print("ReplSetTest awaitReplication: secondary #" + secondaryCount + ", " +
775                               slaveName + ", has config version #" + slaveConfigVersion +
776                               ", but expected config version #" + configVersion);
777 
778                         if (slaveConfigVersion > configVersion) {
779                             master = this.getPrimary();
780                             configVersion =
781                                 master.getDB("local")['system.replset'].findOne().version;
782                             masterOpTime = _getLastOpTimeTimestamp(master);
783                             masterName = master.toString().substr(14);  // strip "connection to "
784 
785                             print("ReplSetTest awaitReplication: timestamp for primary, " +
786                                   masterName + ", is " + tojson(masterLatestOpTime) +
787                                   ", last oplog entry is " + tojsononeline(masterOpTime));
788                         }
789 
790                         return false;
791                     }
792 
793                     // Continue if we're connected to an arbiter
794                     var res = assert.commandWorked(slave.adminCommand({replSetGetStatus: 1}));
795                     if (res.myState == ReplSetTest.State.ARBITER) {
796                         continue;
797                     }
798 
799                     ++secondaryCount;
800                     print("ReplSetTest awaitReplication: checking secondary #" + secondaryCount +
801                           ": " + slaveName);
802 
803                     slave.getDB("admin").getMongo().setSlaveOk();
804 
805                     var getSecondaryTimestampFn = _getLastOpTimeTimestamp;
806                     if (secondaryOpTimeType == ReplSetTest.OpTimeType.LAST_DURABLE) {
807                         getSecondaryTimestampFn = _getDurableOpTimeTimestamp;
808                     }
809 
810                     var ts = getSecondaryTimestampFn(slave);
811                     if (masterLatestOpTime.t < ts.t ||
812                         (masterLatestOpTime.t == ts.t && masterLatestOpTime.i < ts.i)) {
813                         masterLatestOpTime = _getLastOpTimeTimestamp(master);
814                         print("ReplSetTest awaitReplication: timestamp for " + slaveName +
815                               " is newer, resetting latest to " + tojson(masterLatestOpTime));
816                         return false;
817                     }
818 
819                     if (!friendlyEqual(masterLatestOpTime, ts)) {
820                         print("ReplSetTest awaitReplication: timestamp for secondary #" +
821                               secondaryCount + ", " + slaveName + ", is " + tojson(ts) +
822                               " but latest is " + tojson(masterLatestOpTime));
823                         print("ReplSetTest awaitReplication: secondary #" + secondaryCount + ", " +
824                               slaveName + ", is NOT synced");
825                         return false;
826                     }
827 
828                     print("ReplSetTest awaitReplication: secondary #" + secondaryCount + ", " +
829                           slaveName + ", is synced");
830                 }
831 
832                 print("ReplSetTest awaitReplication: finished: all " + secondaryCount +
833                       " secondaries synced at timestamp " + tojson(masterLatestOpTime));
834                 return true;
835             } catch (e) {
836                 print("ReplSetTest awaitReplication: caught exception " + e + ';\n' + e.stack);
837 
838                 // We might have a new master now
839                 awaitLastOpTimeWrittenFn();
840 
841                 print("ReplSetTest awaitReplication: resetting: timestamp for primary " +
842                       self.liveNodes.master + " is " + tojson(masterLatestOpTime));
843 
844                 return false;
845             }
846         }, "awaiting replication", timeout);
847     };
848 
849     this.getHashes = function(db) {
850         this.getPrimary();
851         var res = {};
852         res.master = this.liveNodes.master.getDB(db).runCommand("dbhash");
853         res.slaves = [];
854         this.liveNodes.slaves.forEach(function(node) {
855             var isArbiter = node.getDB('admin').isMaster('admin').arbiterOnly;
856             if (!isArbiter) {
857                 res.slaves.push(node.getDB(db).runCommand("dbhash"));
858             }
859         });
860         return res;
861     };
862 
863     /**
864      * Starts up a server.  Options are saved by default for subsequent starts.
865      *
866      *
867      * Options { remember : true } re-applies the saved options from a prior start.
868      * Options { noRemember : true } ignores the current properties.
869      * Options { appendOptions : true } appends the current options to those remembered.
870      * Options { startClean : true } clears the data directory before starting.
871      *
872      * @param {int|conn|[int|conn]} n array or single server number (0, 1, 2, ...) or conn
873      * @param {object} [options]
874      * @param {boolean} [restart] If false, the data directory will be cleared
875      *   before the server starts.  Default: false.
876      *
877      */
878     this.start = function(n, options, restart, wait) {
879         if (n.length) {
880             var nodes = n;
881             var started = [];
882 
883             for (var i = 0; i < nodes.length; i++) {
884                 if (this.start(nodes[i], Object.merge({}, options), restart, wait)) {
885                     started.push(nodes[i]);
886                 }
887             }
888 
889             return started;
890         }
891 
892         // TODO: should we do something special if we don't currently know about this node?
893         n = this.getNodeId(n);
894 
895         print("ReplSetTest n is : " + n);
896 
897         var defaults = {
898             useHostName: this.useHostName,
899             oplogSize: this.oplogSize,
900             keyFile: this.keyFile,
901             port: _useBridge ? _unbridgedPorts[n] : this.ports[n],
902             noprealloc: "",
903             smallfiles: "",
904             replSet: this.useSeedList ? this.getURL() : this.name,
905             dbpath: "$set-$node"
906         };
907 
908         //
909         // Note : this replaces the binVersion of the shared startSet() options the first time
910         // through, so the full set is guaranteed to have different versions if size > 1.  If using
911         // start() independently, independent version choices will be made
912         //
913         if (options && options.binVersion) {
914             options.binVersion = MongoRunner.versionIterator(options.binVersion);
915         }
916 
917         options = Object.merge(defaults, options);
918         options = Object.merge(options, this.nodeOptions["n" + n]);
919         delete options.rsConfig;
920 
921         options.restart = options.restart || restart;
922 
923         var pathOpts = {node: n, set: this.name};
924         options.pathOpts = Object.merge(options.pathOpts || {}, pathOpts);
925 
926         if (tojson(options) != tojson({}))
927             printjson(options);
928 
929         print("ReplSetTest " + (restart ? "(Re)" : "") + "Starting....");
930 
931         if (_useBridge) {
932             var bridgeOptions = Object.merge(_bridgeOptions, options.bridgeOptions || {});
933             bridgeOptions = Object.merge(bridgeOptions, {
934                 hostName: this.host,
935                 port: this.ports[n],
936                 // The mongod processes identify themselves to mongobridge as host:port, where the
937                 // host is the actual hostname of the machine and not localhost.
938                 dest: getHostName() + ":" + _unbridgedPorts[n],
939             });
940 
941             if (jsTestOptions().networkMessageCompressors) {
942                 bridgeOptions["networkMessageCompressors"] =
943                     jsTestOptions().networkMessageCompressors;
944             }
945 
946             this.nodes[n] = new MongoBridge(bridgeOptions);
947         }
948 
949         var conn = MongoRunner.runMongod(options);
950         if (!conn) {
951             throw new Error("Failed to start node " + n);
952         }
953 
954         // Make sure to call _addPath, otherwise folders won't be cleaned.
955         this._addPath(conn.dbpath);
956 
957         if (_useBridge) {
958             this.nodes[n].connectToBridge();
959             _unbridgedNodes[n] = conn;
960         } else {
961             this.nodes[n] = conn;
962         }
963 
964         // Add replica set specific attributes.
965         this.nodes[n].nodeId = n;
966 
967         printjson(this.nodes);
968 
969         wait = wait || false;
970         if (!wait.toFixed) {
971             if (wait)
972                 wait = 0;
973             else
974                 wait = -1;
975         }
976 
977         if (wait >= 0) {
978             // Wait for node to start up.
979             _waitForHealth(this.nodes[n], Health.UP, wait);
980         }
981 
982         return this.nodes[n];
983     };
984 
985     /**
986      * Restarts a db without clearing the data directory by default.  If the server is not
987      * stopped first, this function will not work.
988      *
989      * Option { startClean : true } forces clearing the data directory.
990      * Option { auth : Object } object that contains the auth details for admin credentials.
991      *   Should contain the fields 'user' and 'pwd'
992      *
993      * @param {int|conn|[int|conn]} n array or single server number (0, 1, 2, ...) or conn
994      */
995     this.restart = function(n, options, signal, wait) {
996         // Can specify wait as third parameter, if using default signal
997         if (signal == true || signal == false) {
998             wait = signal;
999             signal = undefined;
1000         }
1001 
1002         this.stop(n, signal, options);
1003 
1004         var started = this.start(n, options, true, wait);
1005 
1006         if (jsTestOptions().keyFile) {
1007             if (started.length) {
1008                 // if n was an array of conns, start will return an array of connections
1009                 for (var i = 0; i < started.length; i++) {
1010                     assert(jsTest.authenticate(started[i]), "Failed authentication during restart");
1011                 }
1012             } else {
1013                 assert(jsTest.authenticate(started), "Failed authentication during restart");
1014             }
1015         }
1016         return started;
1017     };
1018 
1019     this.stopMaster = function(signal, opts) {
1020         var master = this.getPrimary();
1021         var master_id = this.getNodeId(master);
1022         return this.stop(master_id, signal, opts);
1023     };
1024 
1025     /**
1026      * Stops a particular node or nodes, specified by conn or id
1027      *
1028      * @param {number|Mongo} n the index or connection object of the replica set member to stop.
1029      * @param {number} signal the signal number to use for killing
1030      * @param {Object} opts @see MongoRunner.stopMongod
1031      */
1032     this.stop = function(n, signal, opts) {
1033         // Flatten array of nodes to stop
1034         if (n.length) {
1035             var nodes = n;
1036 
1037             var stopped = [];
1038             for (var i = 0; i < nodes.length; i++) {
1039                 if (this.stop(nodes[i], signal, opts))
1040                     stopped.push(nodes[i]);
1041             }
1042 
1043             return stopped;
1044         }
1045 
1046         // Can specify wait as second parameter, if using default signal
1047         if (signal == true || signal == false) {
1048             signal = undefined;
1049         }
1050 
1051         n = this.getNodeId(n);
1052 
1053         var port = _useBridge ? _unbridgedPorts[n] : this.ports[n];
1054         print('ReplSetTest stop *** Shutting down mongod in port ' + port + ' ***');
1055         var ret = MongoRunner.stopMongod(port, signal, opts);
1056 
1057         print('ReplSetTest stop *** Mongod in port ' + port + ' shutdown with code (' + ret +
1058               ') ***');
1059 
1060         if (_useBridge) {
1061             this.nodes[n].stop();
1062         }
1063 
1064         return ret;
1065     };
1066 
1067     /**
1068      * Kill all members of this replica set.
1069      *
1070      * @param {number} signal The signal number to use for killing the members
1071      * @param {boolean} forRestart will not cleanup data directory
1072      * @param {Object} opts @see MongoRunner.stopMongod
1073      */
1074     this.stopSet = function(signal, forRestart, opts) {
1075         for (var i = 0; i < this.ports.length; i++) {
1076             this.stop(i, signal, opts);
1077         }
1078 
1079         if (forRestart) {
1080             return;
1081         }
1082 
1083         if (_alldbpaths) {
1084             print("ReplSetTest stopSet deleting all dbpaths");
1085             for (var i = 0; i < _alldbpaths.length; i++) {
1086                 resetDbpath(_alldbpaths[i]);
1087             }
1088         }
1089 
1090         _forgetReplSet(this.name);
1091 
1092         print('ReplSetTest stopSet *** Shut down repl set - test worked ****');
1093     };
1094 
1095     /**
1096      * Walks all oplogs and ensures matching entries.
1097      */
1098     this.ensureOplogsMatch = function() {
1099         var OplogReader = function(mongo) {
1100             this.next = function() {
1101                 if (!this.cursor)
1102                     throw Error("reader is not open!");
1103 
1104                 var nextDoc = this.cursor.next();
1105                 if (nextDoc)
1106                     this.lastDoc = nextDoc;
1107                 return nextDoc;
1108             };
1109 
1110             this.getLastDoc = function() {
1111                 if (this.lastDoc)
1112                     return this.lastDoc;
1113                 return this.next();
1114             };
1115 
1116             this.hasNext = function() {
1117                 if (!this.cursor)
1118                     throw Error("reader is not open!");
1119                 return this.cursor.hasNext();
1120             };
1121 
1122             this.query = function(ts) {
1123                 var coll = this.getOplogColl();
1124                 var query = {"ts": {"$gte": ts ? ts : new Timestamp()}};
1125                 this.cursor = coll.find(query).sort({$natural: 1});
1126                 this.cursor.addOption(DBQuery.Option.oplogReplay);
1127             };
1128 
1129             this.getFirstDoc = function() {
1130                 return this.getOplogColl().find().sort({$natural: 1}).limit(-1).next();
1131             };
1132 
1133             this.getOplogColl = function() {
1134                 return this.mongo.getDB("local")["oplog.rs"];
1135             };
1136 
1137             this.lastDoc = null;
1138             this.cursor = null;
1139             this.mongo = mongo;
1140         };
1141 
1142         if (this.nodes.length && this.nodes.length > 1) {
1143             var readers = [];
1144             var largestTS = null;
1145             var nodes = this.nodes;
1146             var rsSize = nodes.length;
1147             for (var i = 0; i < rsSize; i++) {
1148                 readers[i] = new OplogReader(nodes[i]);
1149                 var currTS = readers[i].getFirstDoc().ts;
1150                 if (currTS.t > largestTS.t || (currTS.t == largestTS.t && currTS.i > largestTS.i)) {
1151                     largestTS = currTS;
1152                 }
1153             }
1154 
1155             // start all oplogReaders at the same place.
1156             for (i = 0; i < rsSize; i++) {
1157                 readers[i].query(largestTS);
1158             }
1159 
1160             var firstReader = readers[0];
1161             while (firstReader.hasNext()) {
1162                 var ts = firstReader.next().ts;
1163                 for (i = 1; i < rsSize; i++) {
1164                     assert.eq(
1165                         ts, readers[i].next().ts, " non-matching ts for node: " + readers[i].mongo);
1166                 }
1167             }
1168 
1169             // ensure no other node has more oplog
1170             for (i = 1; i < rsSize; i++) {
1171                 assert.eq(
1172                     false, readers[i].hasNext(), "" + readers[i] + " shouldn't have more oplog.");
1173             }
1174         }
1175     };
1176 
1177     /**
1178      * Wait for a state indicator to go to a particular state or states.
1179      *
1180      * @param node is a single node or list of nodes, by id or conn
1181      * @param state is a single state or list of states
1182      *
1183      */
1184     this.waitForState = function(node, state, timeout) {
1185         _waitForIndicator(node, state, "state", timeout);
1186     };
1187 
1188     /**
1189      * Waits until there is a master node.
1190      */
1191     this.waitForMaster = function(timeout) {
1192         var master;
1193         assert.soonNoExcept(function() {
1194             return (master = self.getPrimary());
1195         }, "waiting for master", timeout);
1196 
1197         return master;
1198     };
1199 
1200     //
1201     // ReplSetTest constructors
1202     //
1203 
1204     /**
1205      * Constructor, which initializes the ReplSetTest object by starting new instances.
1206      */
1207     function _constructStartNewInstances(opts) {
1208         self.name = opts.name || "testReplSet";
1209         print('Starting new replica set ' + self.name);
1210 
1211         self.useHostName = opts.useHostName == undefined ? true : opts.useHostName;
1212         self.host = self.useHostName ? (opts.host || getHostName()) : 'localhost';
1213         self.oplogSize = opts.oplogSize || 40;
1214         self.useSeedList = opts.useSeedList || false;
1215         self.keyFile = opts.keyFile;
1216         self.protocolVersion = opts.protocolVersion;
1217 
1218         _useBridge = opts.useBridge || false;
1219         _bridgeOptions = opts.bridgeOptions || {};
1220 
1221         _configSettings = opts.settings || false;
1222 
1223         self.nodeOptions = {};
1224 
1225         var numNodes;
1226 
1227         if (isObject(opts.nodes)) {
1228             var len = 0;
1229             for (var i in opts.nodes) {
1230                 var options = self.nodeOptions["n" + len] =
1231                     Object.merge(opts.nodeOptions, opts.nodes[i]);
1232                 if (i.startsWith("a")) {
1233                     options.arbiter = true;
1234                 }
1235 
1236                 len++;
1237             }
1238 
1239             numNodes = len;
1240         } else if (Array.isArray(opts.nodes)) {
1241             for (var i = 0; i < opts.nodes.length; i++) {
1242                 self.nodeOptions["n" + i] = Object.merge(opts.nodeOptions, opts.nodes[i]);
1243             }
1244 
1245             numNodes = opts.nodes.length;
1246         } else {
1247             for (var i = 0; i < opts.nodes; i++) {
1248                 self.nodeOptions["n" + i] = opts.nodeOptions;
1249             }
1250 
1251             numNodes = opts.nodes;
1252         }
1253 
1254         self.ports = allocatePorts(numNodes);
1255         self.nodes = [];
1256 
1257         if (_useBridge) {
1258             _unbridgedPorts = allocatePorts(numNodes);
1259             _unbridgedNodes = [];
1260         }
1261     }
1262 
1263     /**
1264      * Constructor, which instantiates the ReplSetTest object from an existing set.
1265      */
1266     function _constructFromExistingSeedNode(seedNode) {
1267         var conf = _replSetGetConfig(new Mongo(seedNode));
1268         print('Recreating replica set from config ' + tojson(conf));
1269 
1270         var existingNodes = conf.members.map(member => member.host);
1271         self.ports = existingNodes.map(node => node.split(':')[1]);
1272         self.nodes = existingNodes.map(node => new Mongo(node));
1273     }
1274 
1275     if (typeof opts === 'string' || opts instanceof String) {
1276         _constructFromExistingSeedNode(opts);
1277     } else {
1278         _constructStartNewInstances(opts);
1279     }
1280 };
1281 
1282 /**
1283  * Set of states that the replica set can be in. Used for the wait functions.
1284  */
1285 ReplSetTest.State = {
1286     PRIMARY: 1,
1287     SECONDARY: 2,
1288     RECOVERING: 3,
1289     // Note there is no state 4
1290     STARTUP_2: 5,
1291     UNKNOWN: 6,
1292     ARBITER: 7,
1293     DOWN: 8,
1294     ROLLBACK: 9,
1295     REMOVED: 10,
1296 };
1297 
1298 ReplSetTest.OpTimeType = {
1299     LAST_APPLIED: 1,
1300     LAST_DURABLE: 2,
1301 };
1302 
1303 /**
1304  * Waits for the specified hosts to enter a certain state.
1305  */
1306 ReplSetTest.awaitRSClientHosts = function(conn, host, hostOk, rs, timeout) {
1307     var hostCount = host.length;
1308     if (hostCount) {
1309         for (var i = 0; i < hostCount; i++) {
1310             ReplSetTest.awaitRSClientHosts(conn, host[i], hostOk, rs);
1311         }
1312 
1313         return;
1314     }
1315 
1316     timeout = timeout || 60000;
1317 
1318     if (hostOk == undefined)
1319         hostOk = {ok: true};
1320     if (host.host)
1321         host = host.host;
1322     if (rs)
1323         rs = rs.name;
1324 
1325     print("Awaiting " + host + " to be " + tojson(hostOk) + " for " + conn + " (rs: " + rs + ")");
1326 
1327     var tests = 0;
1328 
1329     assert.soon(function() {
1330         var rsClientHosts = conn.adminCommand('connPoolStats').replicaSets;
1331         if (tests++ % 10 == 0) {
1332             printjson(rsClientHosts);
1333         }
1334 
1335         for (var rsName in rsClientHosts) {
1336             if (rs && rs != rsName)
1337                 continue;
1338 
1339             for (var i = 0; i < rsClientHosts[rsName].hosts.length; i++) {
1340                 var clientHost = rsClientHosts[rsName].hosts[i];
1341                 if (clientHost.addr != host)
1342                     continue;
1343 
1344                 // Check that *all* host properties are set correctly
1345                 var propOk = true;
1346                 for (var prop in hostOk) {
1347                     // Use special comparator for tags because isMaster can return the fields in
1348                     // different order. The fields of the tags should be treated like a set of
1349                     // strings and 2 tags should be considered the same if the set is equal.
1350                     if (prop == 'tags') {
1351                         if (!clientHost.tags) {
1352                             propOk = false;
1353                             break;
1354                         }
1355 
1356                         for (var hostTag in hostOk.tags) {
1357                             if (clientHost.tags[hostTag] != hostOk.tags[hostTag]) {
1358                                 propOk = false;
1359                                 break;
1360                             }
1361                         }
1362 
1363                         for (var clientTag in clientHost.tags) {
1364                             if (clientHost.tags[clientTag] != hostOk.tags[clientTag]) {
1365                                 propOk = false;
1366                                 break;
1367                             }
1368                         }
1369 
1370                         continue;
1371                     }
1372 
1373                     if (isObject(hostOk[prop])) {
1374                         if (!friendlyEqual(hostOk[prop], clientHost[prop])) {
1375                             propOk = false;
1376                             break;
1377                         }
1378                     } else if (clientHost[prop] != hostOk[prop]) {
1379                         propOk = false;
1380                         break;
1381                     }
1382                 }
1383 
1384                 if (propOk) {
1385                     return true;
1386                 }
1387             }
1388         }
1389 
1390         return false;
1391     }, 'timed out waiting for replica set client to recognize hosts', timeout);
1392 };
1393