1 /**
  2  * Starts up a sharded cluster with the given specifications. The cluster will be fully operational
  3  * after the execution of this constructor function.
  4  *
  5  * In addition to its own methods, ShardingTest inherits all the functions from the 'sh' utility
  6  * with the db set as the first mongos instance in the test (i.e. s0).
  7  *
  8  * @param {Object} params Contains the key-value pairs for the cluster
  9  *   configuration. Accepted keys are:
 10  *
 11  *   {
 12  *     name {string}: name for this test
 13  *     verbose {number}: the verbosity for the mongos
 14  *     chunkSize {number}: the chunk size to use as configuration for the cluster
 15  *     nopreallocj {boolean|number}:
 16  *
 17  *     mongos {number|Object|Array.<Object>}: number of mongos or mongos
 18  *       configuration object(s)(*). @see MongoRunner.runMongos
 19  *
 20  *     rs {Object|Array.<Object>}: replica set configuration object. Can
 21  *       contain:
 22  *       {
 23  *         nodes {number}: number of replica members. Defaults to 3.
 24  *         protocolVersion {number}: protocol version of replset used by the
 25  *             replset initiation.
 26  *         initiateTimeout {number}: timeout in milliseconds to specify
 27  *              to ReplSetTest.prototype.initiate().
 28  *         For other options, @see ReplSetTest#start
 29  *       }
 30  *
 31  *     shards {number|Object|Array.<Object>}: number of shards or shard
 32  *       configuration object(s)(*). @see MongoRunner.runMongod
 33  *
 34  *     config {number|Object|Array.<Object>}: number of config server or
 35  *       config server configuration object(s)(*). @see MongoRunner.runMongod
 36  *
 37  *     (*) There are two ways For multiple configuration objects.
 38  *       (1) Using the object format. Example:
 39  *
 40  *           { d0: { verbose: 5 }, d1: { auth: '' }, rs2: { oplogsize: 10 }}
 41  *
 42  *           In this format, d = mongod, s = mongos & c = config servers
 43  *
 44  *       (2) Using the array format. Example:
 45  *
 46  *           [{ verbose: 5 }, { auth: '' }]
 47  *
 48  *       Note: you can only have single server shards for array format.
 49  *
 50  *       Note: A special "bridgeOptions" property can be specified in both the object and array
 51  *          formats to configure the options for the mongobridge corresponding to that node. These
 52  *          options are merged with the params.bridgeOptions options, where the node-specific
 53  *          options take precedence.
 54  *
 55  *     other: {
 56  *       nopreallocj: same as above
 57  *       rs: same as above
 58  *       chunkSize: same as above
 59  *       keyFile {string}: the location of the keyFile
 60  *
 61  *       shardOptions {Object}: same as the shards property above.
 62  *          Can be used to specify options that are common all shards.
 63  *
 64  *       configOptions {Object}: same as the config property above.
 65  *          Can be used to specify options that are common all config servers.
 66  *       mongosOptions {Object}: same as the mongos property above.
 67  *          Can be used to specify options that are common all mongos.
 68  *       enableBalancer {boolean} : if true, enable the balancer
 69  *       manualAddShard {boolean}: shards will not be added if true.
 70  *
 71  *       useBridge {boolean}: If true, then a mongobridge process is started for each node in the
 72  *          sharded cluster. Defaults to false.
 73  *
 74  *       bridgeOptions {Object}: Options to apply to all mongobridge processes. Defaults to {}.
 75  *
 76  *       // replica Set only:
 77  *       rsOptions {Object}: same as the rs property above. Can be used to
 78  *         specify options that are common all replica members.
 79  *       useHostname {boolean}: if true, use hostname of machine,
 80  *         otherwise use localhost
 81  *       numReplicas {number},
 82  *       waitForCSRSSecondaries {boolean}: if false, will not wait for the read committed view
 83  *         of the secondaries to catch up with the primary. Defaults to true.
 84  *     }
 85  *   }
 86  *
 87  * Member variables:
 88  * s {Mongo} - connection to the first mongos
 89  * s0, s1, ... {Mongo} - connection to different mongos
 90  * rs0, rs1, ... {ReplSetTest} - test objects to replica sets
 91  * shard0, shard1, ... {Mongo} - connection to shards (not available for replica sets)
 92  * d0, d1, ... {Mongo} - same as shard0, shard1, ...
 93  * config0, config1, ... {Mongo} - connection to config servers
 94  * c0, c1, ... {Mongo} - same as config0, config1, ...
 95  * configRS - If the config servers are a replset, this will contain the config ReplSetTest object
 96  */
 97 var ShardingTest = function(params) {
 98 
 99     if (!(this instanceof ShardingTest)) {
100         return new ShardingTest(params);
101     }
102 
103     // Capture the 'this' reference
104     var self = this;
105 
106     // Used for counting the test duration
107     var _startTime = new Date();
108 
109     // Populated with the paths of all shard hosts (config servers + hosts) and is used for
110     // cleaning up the data files on shutdown
111     var _alldbpaths = [];
112 
113     // Publicly exposed variables
114 
115     /**
116      * Attempts to open a connection to the specified connection string or throws if unable to
117      * connect.
118      */
119     function _connectWithRetry(url) {
120         var conn;
121         assert.soon(function() {
122             try {
123                 conn = new Mongo(url);
124                 return true;
125             } catch (e) {
126                 print("Error connecting to " + url + ": " + e);
127                 return false;
128             }
129         });
130 
131         return conn;
132     }
133 
134     /**
135      * Constructs a human-readable string representing a chunk's range.
136      */
137     function _rangeToString(r) {
138         return tojsononeline(r.min) + " -> " + tojsononeline(r.max);
139     }
140 
141     /**
142      * Checks whether the specified collection is sharded by consulting the config metadata.
143      */
144     function _isSharded(collName) {
145         var collName = "" + collName;
146         var dbName;
147 
148         if (typeof collName.getCollectionNames == 'function') {
149             dbName = "" + collName;
150             collName = undefined;
151         }
152 
153         if (dbName) {
154             var x = self.config.databases.findOne({_id: dbname});
155             if (x)
156                 return x.partitioned;
157             else
158                 return false;
159         }
160 
161         if (collName) {
162             var x = self.config.collections.findOne({_id: collName});
163             if (x)
164                 return true;
165             else
166                 return false;
167         }
168     }
169 
170     /**
171      * Extends the ShardingTest class with the methods exposed by the sh utility class.
172      */
173     function _extendWithShMethods() {
174         Object.keys(sh).forEach(function(fn) {
175             if (typeof sh[fn] !== 'function') {
176                 return;
177             }
178 
179             assert.eq(undefined,
180                       self[fn],
181                       'ShardingTest contains a method ' + fn +
182                           ' which duplicates a method with the same name on sh. ' +
183                           'Please select a different function name.');
184 
185             self[fn] = function() {
186                 if (typeof db == "undefined") {
187                     db = undefined;
188                 }
189 
190                 var oldDb = db;
191                 db = self.getDB('test');
192 
193                 try {
194                     return sh[fn].apply(sh, arguments);
195                 } finally {
196                     db = oldDb;
197                 }
198             };
199         });
200     }
201 
202     /**
203      * Configures the cluster based on the specified parameters (balancer state, etc).
204      */
205     function _configureCluster() {
206         // Disable the balancer unless it is explicitly turned on
207         if (!otherParams.enableBalancer) {
208             self.stopBalancer();
209         }
210     }
211 
212     function connectionURLTheSame(a, b) {
213         if (a == b)
214             return true;
215 
216         if (!a || !b)
217             return false;
218 
219         if (a.host)
220             return connectionURLTheSame(a.host, b);
221         if (b.host)
222             return connectionURLTheSame(a, b.host);
223 
224         if (a.name)
225             return connectionURLTheSame(a.name, b);
226         if (b.name)
227             return connectionURLTheSame(a, b.name);
228 
229         if (a.indexOf("/") < 0 && b.indexOf("/") < 0) {
230             a = a.split(":");
231             b = b.split(":");
232 
233             if (a.length != b.length)
234                 return false;
235 
236             if (a.length == 2 && a[1] != b[1])
237                 return false;
238 
239             if (a[0] == "localhost" || a[0] == "127.0.0.1")
240                 a[0] = getHostName();
241             if (b[0] == "localhost" || b[0] == "127.0.0.1")
242                 b[0] = getHostName();
243 
244             return a[0] == b[0];
245         } else {
246             var a0 = a.split("/")[0];
247             var b0 = b.split("/")[0];
248             return a0 == b0;
249         }
250     }
251 
252     assert(connectionURLTheSame("foo", "foo"));
253     assert(!connectionURLTheSame("foo", "bar"));
254 
255     assert(connectionURLTheSame("foo/a,b", "foo/b,a"));
256     assert(!connectionURLTheSame("foo/a,b", "bar/a,b"));
257 
258     // ShardingTest API
259 
260     this.getDB = function(name) {
261         return this.s.getDB(name);
262     };
263 
264     /**
265      * Finds the _id of the primary shard for database 'dbname', e.g., 'test-rs0'
266      */
267     this.getPrimaryShardIdForDatabase = function(dbname) {
268         var x = this.config.databases.findOne({_id: "" + dbname});
269         if (x) {
270             return x.primary;
271         }
272 
273         var countDBsFound = 0;
274         this.config.databases.find().forEach(function(db) {
275             countDBsFound++;
276             printjson(db);
277         });
278         throw Error("couldn't find dbname: " + dbname + " in config.databases. Total DBs: " +
279                     countDBsFound);
280     };
281 
282     this.getNonPrimaries = function(dbname) {
283         var x = this.config.databases.findOne({_id: dbname});
284         if (!x) {
285             this.config.databases.find().forEach(printjson);
286             throw Error("couldn't find dbname: " + dbname + " total: " +
287                         this.config.databases.count());
288         }
289 
290         return this.config.shards.find({_id: {$ne: x.primary}}).map(z => z._id);
291     };
292 
293     this.getConnNames = function() {
294         var names = [];
295         for (var i = 0; i < this._connections.length; i++) {
296             names.push(this._connections[i].name);
297         }
298         return names;
299     };
300 
301     /**
302      * Find the connection to the primary shard for database 'dbname'.
303      */
304     this.getPrimaryShard = function(dbname) {
305         var dbPrimaryShardId = this.getPrimaryShardIdForDatabase(dbname);
306         var primaryShard = this.config.shards.findOne({_id: dbPrimaryShardId});
307 
308         if (primaryShard) {
309             shardConnectionString = primaryShard.host;
310             var rsName = shardConnectionString.substring(0, shardConnectionString.indexOf("/"));
311 
312             for (var i = 0; i < this._connections.length; i++) {
313                 var c = this._connections[i];
314                 if (connectionURLTheSame(shardConnectionString, c.name) ||
315                     connectionURLTheSame(rsName, c.name))
316                     return c;
317             }
318         }
319 
320         throw Error("can't find server connection for db '" + dbname + "'s primary shard: " +
321                     tojson(primaryShard));
322     };
323 
324     this.normalize = function(x) {
325         var z = this.config.shards.findOne({host: x});
326         if (z)
327             return z._id;
328         return x;
329     };
330 
331     /**
332      * Find a different shard connection than the one given.
333      */
334     this.getOther = function(one) {
335         if (this._connections.length < 2) {
336             throw Error("getOther only works with 2 shards");
337         }
338 
339         if (one._mongo) {
340             one = one._mongo;
341         }
342 
343         for (var i = 0; i < this._connections.length; i++) {
344             if (this._connections[i] != one) {
345                 return this._connections[i];
346             }
347         }
348 
349         return null;
350     };
351 
352     this.getAnother = function(one) {
353         if (this._connections.length < 2) {
354             throw Error("getAnother() only works with multiple servers");
355         }
356 
357         if (one._mongo) {
358             one = one._mongo;
359         }
360 
361         for (var i = 0; i < this._connections.length; i++) {
362             if (this._connections[i] == one)
363                 return this._connections[(i + 1) % this._connections.length];
364         }
365     };
366 
367     this.stop = function(opts) {
368         for (var i = 0; i < this._mongos.length; i++) {
369             this.stopMongos(i, opts);
370         }
371 
372         for (var i = 0; i < this._connections.length; i++) {
373             if (this._rs[i]) {
374                 this._rs[i].test.stopSet(15, undefined, opts);
375             } else {
376                 this.stopMongod(i, opts);
377             }
378         }
379 
380         if (this.configRS) {
381             this.configRS.stopSet(undefined, undefined, opts);
382         } else {
383             // Old style config triplet
384             for (var i = 0; i < this._configServers.length; i++) {
385                 this.stopConfigServer(i, opts);
386             }
387         }
388 
389         for (var i = 0; i < _alldbpaths.length; i++) {
390             resetDbpath(MongoRunner.dataPath + _alldbpaths[i]);
391         }
392 
393         var timeMillis = new Date().getTime() - _startTime.getTime();
394 
395         print('*** ShardingTest ' + this._testName + " completed successfully in " +
396               (timeMillis / 1000) + " seconds ***");
397     };
398 
399     this.getDBPaths = function() {
400         return _alldbpaths.map((path) => {
401             return MongoRunner.dataPath + path;
402         });
403     };
404 
405     this.adminCommand = function(cmd) {
406         var res = this.admin.runCommand(cmd);
407         if (res && res.ok == 1)
408             return true;
409 
410         throw _getErrorWithCode(res, "command " + tojson(cmd) + " failed: " + tojson(res));
411     };
412 
413     this.printChangeLog = function() {
414         this.config.changelog.find().forEach(function(z) {
415             var msg = z.server + "\t" + z.time + "\t" + z.what;
416             for (var i = z.what.length; i < 15; i++)
417                 msg += " ";
418 
419             msg += " " + z.ns + "\t";
420             if (z.what == "split") {
421                 msg += _rangeToString(z.details.before) + " -->> (" +
422                     _rangeToString(z.details.left) + "), (" + _rangeToString(z.details.right) + ")";
423             } else if (z.what == "multi-split") {
424                 msg += _rangeToString(z.details.before) + "  -->> (" + z.details.number + "/" +
425                     z.details.of + " " + _rangeToString(z.details.chunk) + ")";
426             } else {
427                 msg += tojsononeline(z.details);
428             }
429 
430             print("ShardingTest " + msg);
431         });
432     };
433 
434     this.getChunksString = function(ns) {
435         var q = {};
436         if (ns) {
437             q.ns = ns;
438         }
439 
440         var s = "";
441         this.config.chunks.find(q).sort({ns: 1, min: 1}).forEach(function(z) {
442             s += "  " + z._id + "\t" + z.lastmod.t + "|" + z.lastmod.i + "\t" + tojson(z.min) +
443                 " -> " + tojson(z.max) + " " + z.shard + "  " + z.ns + "\n";
444         });
445 
446         return s;
447     };
448 
449     this.printChunks = function(ns) {
450         print("ShardingTest " + this.getChunksString(ns));
451     };
452 
453     this.printShardingStatus = function(verbose) {
454         printShardingStatus(this.config, verbose);
455     };
456 
457     this.printCollectionInfo = function(ns, msg) {
458         var out = "";
459         if (msg) {
460             out += msg + "\n";
461         }
462         out += "sharding collection info: " + ns + "\n";
463 
464         for (var i = 0; i < this._connections.length; i++) {
465             var c = this._connections[i];
466             out += "  mongod " + c + " " +
467                 tojson(c.getCollection(ns).getShardVersion(), " ", true) + "\n";
468         }
469 
470         for (var i = 0; i < this._mongos.length; i++) {
471             var c = this._mongos[i];
472             out += "  mongos " + c + " " +
473                 tojson(c.getCollection(ns).getShardVersion(), " ", true) + "\n";
474         }
475 
476         out += this.getChunksString(ns);
477 
478         print("ShardingTest " + out);
479     };
480 
481     this.sync = function() {
482         this.adminCommand("connpoolsync");
483     };
484 
485     this.onNumShards = function(collName, dbName) {
486         dbName = dbName || "test";
487 
488         // We should sync since we're going directly to mongod here
489         this.sync();
490 
491         var num = 0;
492         for (var i = 0; i < this._connections.length; i++) {
493             if (this._connections[i].getDB(dbName).getCollection(collName).count() > 0) {
494                 num++;
495             }
496         }
497 
498         return num;
499     };
500 
501     this.shardCounts = function(collName, dbName) {
502         dbName = dbName || "test";
503 
504         // We should sync since we're going directly to mongod here
505         this.sync();
506 
507         var counts = {};
508         for (var i = 0; i < this._connections.length; i++) {
509             counts[i] = this._connections[i].getDB(dbName).getCollection(collName).count();
510         }
511 
512         return counts;
513     };
514 
515     this.chunkCounts = function(collName, dbName) {
516         dbName = dbName || "test";
517 
518         var x = {};
519         this.config.shards.find().forEach(function(z) {
520             x[z._id] = 0;
521         });
522 
523         this.config.chunks.find({ns: dbName + "." + collName}).forEach(function(z) {
524             if (x[z.shard])
525                 x[z.shard]++;
526             else
527                 x[z.shard] = 1;
528         });
529 
530         return x;
531     };
532 
533     this.chunkDiff = function(collName, dbName) {
534         var c = this.chunkCounts(collName, dbName);
535 
536         var min = 100000000;
537         var max = 0;
538         for (var s in c) {
539             if (c[s] < min)
540                 min = c[s];
541             if (c[s] > max)
542                 max = c[s];
543         }
544 
545         print("ShardingTest input: " + tojson(c) + " min: " + min + " max: " + max);
546         return max - min;
547     };
548 
549     /**
550      * Waits up to the specified timeout (with a default of 60s) for the balancer to execute one
551      * round. If no round has been executed, throws an error.
552      *
553      * The mongosConnection parameter is optional and allows callers to specify a connection
554      * different than the first mongos instance in the list.
555      */
556     this.awaitBalancerRound = function(timeoutMs, mongosConnection) {
557         timeoutMs = timeoutMs || 60000;
558         mongosConnection = mongosConnection || self.s0;
559 
560         // Get the balancer section from the server status of the config server primary
561         function getBalancerStatus() {
562             var balancerStatus =
563                 assert.commandWorked(mongosConnection.adminCommand({balancerStatus: 1}));
564             if (balancerStatus.mode !== 'full') {
565                 throw Error('Balancer is not enabled');
566             }
567 
568             return balancerStatus;
569         }
570 
571         var initialStatus = getBalancerStatus();
572         var currentStatus;
573         assert.soon(function() {
574             currentStatus = getBalancerStatus();
575             return (currentStatus.numBalancerRounds - initialStatus.numBalancerRounds) != 0;
576         }, 'Latest balancer status' + currentStatus, timeoutMs);
577     };
578 
579     /**
580      * Waits up to one minute for the difference in chunks between the most loaded shard and
581      * least loaded shard to be 0 or 1, indicating that the collection is well balanced. This should
582      * only be called after creating a big enough chunk difference to trigger balancing.
583      */
584     this.awaitBalance = function(collName, dbName, timeToWait) {
585         timeToWait = timeToWait || 60000;
586 
587         assert.soon(function() {
588             var x = self.chunkDiff(collName, dbName);
589             print("chunk diff: " + x);
590             return x < 2;
591         }, "no balance happened", timeToWait);
592     };
593 
594     this.getShard = function(coll, query, includeEmpty) {
595         var shards = this.getShardsForQuery(coll, query, includeEmpty);
596         assert.eq(shards.length, 1);
597         return shards[0];
598     };
599 
600     /**
601      * Returns the shards on which documents matching a particular query reside.
602      */
603     this.getShardsForQuery = function(coll, query, includeEmpty) {
604         if (!coll.getDB) {
605             coll = this.s.getCollection(coll);
606         }
607 
608         var explain = coll.find(query).explain("executionStats");
609         var shards = [];
610 
611         var execStages = explain.executionStats.executionStages;
612         var plannerShards = explain.queryPlanner.winningPlan.shards;
613 
614         if (execStages.shards) {
615             for (var i = 0; i < execStages.shards.length; i++) {
616                 var hasResults = execStages.shards[i].executionStages.nReturned &&
617                     execStages.shards[i].executionStages.nReturned > 0;
618                 if (includeEmpty || hasResults) {
619                     shards.push(plannerShards[i].connectionString);
620                 }
621             }
622         }
623 
624         for (var i = 0; i < shards.length; i++) {
625             for (var j = 0; j < this._connections.length; j++) {
626                 if (connectionURLTheSame(this._connections[j], shards[i])) {
627                     shards[i] = this._connections[j];
628                     break;
629                 }
630             }
631         }
632 
633         return shards;
634     };
635 
636     this.shardColl = function(collName, key, split, move, dbName, waitForDelete) {
637         split = (split != false ? (split || key) : split);
638         move = (split != false && move != false ? (move || split) : false);
639 
640         if (collName.getDB)
641             dbName = "" + collName.getDB();
642         else
643             dbName = dbName || "test";
644 
645         var c = dbName + "." + collName;
646         if (collName.getDB) {
647             c = "" + collName;
648         }
649 
650         var isEmpty = (this.s.getCollection(c).count() == 0);
651 
652         if (!_isSharded(dbName)) {
653             this.s.adminCommand({enableSharding: dbName});
654         }
655 
656         var result = this.s.adminCommand({shardcollection: c, key: key});
657         if (!result.ok) {
658             printjson(result);
659             assert(false);
660         }
661 
662         if (split == false) {
663             return;
664         }
665 
666         result = this.s.adminCommand({split: c, middle: split});
667         if (!result.ok) {
668             printjson(result);
669             assert(false);
670         }
671 
672         if (move == false) {
673             return;
674         }
675 
676         var result;
677         for (var i = 0; i < 5; i++) {
678             var otherShard = this.getOther(this.getPrimaryShard(dbName)).name;
679             result = this.s.adminCommand(
680                 {movechunk: c, find: move, to: otherShard, _waitForDelete: waitForDelete});
681             if (result.ok)
682                 break;
683 
684             sleep(5 * 1000);
685         }
686 
687         printjson(result);
688         assert(result.ok);
689     };
690 
691     /**
692      * Kills the mongos with index n.
693      */
694     this.stopMongos = function(n, opts) {
695         if (otherParams.useBridge) {
696             MongoRunner.stopMongos(unbridgedMongos[n], undefined, opts);
697             this["s" + n].stop();
698         } else {
699             MongoRunner.stopMongos(this["s" + n], undefined, opts);
700         }
701     };
702 
703     /**
704      * Kills the shard mongod with index n.
705      */
706     this.stopMongod = function(n, opts) {
707         if (otherParams.useBridge) {
708             MongoRunner.stopMongod(unbridgedConnections[n], undefined, opts);
709             this["d" + n].stop();
710         } else {
711             MongoRunner.stopMongod(this["d" + n], undefined, opts);
712         }
713     };
714 
715     /**
716      * Kills the config server mongod with index n.
717      */
718     this.stopConfigServer = function(n, opts) {
719         if (otherParams.useBridge) {
720             MongoRunner.stopMongod(unbridgedConfigServers[n], undefined, opts);
721             this._configServers[n].stop();
722         } else {
723             MongoRunner.stopMongod(this._configServers[n], undefined, opts);
724         }
725     };
726 
727     /**
728      * Stops and restarts a mongos process.
729      *
730      * If opts is specified, the new mongos is started using those options. Otherwise, it is started
731      * with its previous parameters.
732      *
733      * Warning: Overwrites the old s (if n = 0) admin, config, and sn member variables.
734      */
735     this.restartMongos = function(n, opts) {
736         var mongos;
737 
738         if (otherParams.useBridge) {
739             mongos = unbridgedMongos[n];
740         } else {
741             mongos = this["s" + n];
742         }
743 
744         opts = opts || mongos;
745         opts.port = opts.port || mongos.port;
746 
747         this.stopMongos(n);
748 
749         if (otherParams.useBridge) {
750             var bridgeOptions =
751                 (opts !== mongos) ? opts.bridgeOptions : mongos.fullOptions.bridgeOptions;
752             bridgeOptions = Object.merge(otherParams.bridgeOptions, bridgeOptions || {});
753             bridgeOptions = Object.merge(bridgeOptions, {
754                 hostName: otherParams.useHostname ? hostName : "localhost",
755                 port: this._mongos[n].port,
756                 // The mongos processes identify themselves to mongobridge as host:port, where the
757                 // host is the actual hostname of the machine and not localhost.
758                 dest: hostName + ":" + opts.port,
759             });
760 
761             this._mongos[n] = new MongoBridge(bridgeOptions);
762         }
763 
764         var newConn = MongoRunner.runMongos(opts);
765         if (!newConn) {
766             throw new Error("Failed to restart mongos " + n);
767         }
768 
769         if (otherParams.useBridge) {
770             this._mongos[n].connectToBridge();
771             unbridgedMongos[n] = newConn;
772         } else {
773             this._mongos[n] = newConn;
774         }
775 
776         this['s' + n] = this._mongos[n];
777         if (n == 0) {
778             this.s = this._mongos[n];
779             this.admin = this._mongos[n].getDB('admin');
780             this.config = this._mongos[n].getDB('config');
781         }
782     };
783 
784     /**
785      * Stops and restarts a shard mongod process.
786      *
787      * If opts is specified, the new mongod is started using those options. Otherwise, it is started
788      * with its previous parameters. The 'beforeRestartCallback' parameter is an optional function
789      * that will be run after the MongoD is stopped, but before it is restarted. The intended uses
790      * of the callback are modifications to the dbpath of the mongod that must be made while it is
791      * stopped.
792      *
793      * Warning: Overwrites the old dn/shardn member variables.
794      */
795     this.restartMongod = function(n, opts, beforeRestartCallback) {
796         var mongod;
797 
798         if (otherParams.useBridge) {
799             mongod = unbridgedConnections[n];
800         } else {
801             mongod = this["d" + n];
802         }
803 
804         opts = opts || mongod;
805         opts.port = opts.port || mongod.port;
806 
807         this.stopMongod(n);
808 
809         if (otherParams.useBridge) {
810             var bridgeOptions =
811                 (opts !== mongod) ? opts.bridgeOptions : mongod.fullOptions.bridgeOptions;
812             bridgeOptions = Object.merge(otherParams.bridgeOptions, bridgeOptions || {});
813             bridgeOptions = Object.merge(bridgeOptions, {
814                 hostName: otherParams.useHostname ? hostName : "localhost",
815                 port: this._connections[n].port,
816                 // The mongod processes identify themselves to mongobridge as host:port, where the
817                 // host is the actual hostname of the machine and not localhost.
818                 dest: hostName + ":" + opts.port,
819             });
820 
821             this._connections[n] = new MongoBridge(bridgeOptions);
822         }
823 
824         if (arguments.length >= 3) {
825             if (typeof(beforeRestartCallback) !== "function") {
826                 throw new Error("beforeRestartCallback must be a function but was of type " +
827                                 typeof(beforeRestartCallback));
828             }
829             beforeRestartCallback();
830         }
831 
832         opts.restart = true;
833 
834         var newConn = MongoRunner.runMongod(opts);
835         if (!newConn) {
836             throw new Error("Failed to restart shard " + n);
837         }
838 
839         if (otherParams.useBridge) {
840             this._connections[n].connectToBridge();
841             unbridgedConnections[n] = newConn;
842         } else {
843             this._connections[n] = newConn;
844         }
845 
846         this["shard" + n] = this._connections[n];
847         this["d" + n] = this._connections[n];
848     };
849 
850     /**
851      * Stops and restarts a config server mongod process.
852      *
853      * If opts is specified, the new mongod is started using those options. Otherwise, it is started
854      * with its previous parameters.
855      *
856      * Warning: Overwrites the old cn/confign member variables.
857      */
858     this.restartConfigServer = function(n) {
859         var mongod;
860 
861         if (otherParams.useBridge) {
862             mongod = unbridgedConfigServers[n];
863         } else {
864             mongod = this["c" + n];
865         }
866 
867         this.stopConfigServer(n);
868 
869         if (otherParams.useBridge) {
870             var bridgeOptions =
871                 Object.merge(otherParams.bridgeOptions, mongod.fullOptions.bridgeOptions || {});
872             bridgeOptions = Object.merge(bridgeOptions, {
873                 hostName: otherParams.useHostname ? hostName : "localhost",
874                 port: this._configServers[n].port,
875                 // The mongod processes identify themselves to mongobridge as host:port, where the
876                 // host is the actual hostname of the machine and not localhost.
877                 dest: hostName + ":" + mongod.port,
878             });
879 
880             this._configServers[n] = new MongoBridge(bridgeOptions);
881         }
882 
883         mongod.restart = true;
884         var newConn = MongoRunner.runMongod(mongod);
885         if (!newConn) {
886             throw new Error("Failed to restart config server " + n);
887         }
888 
889         if (otherParams.useBridge) {
890             this._configServers[n].connectToBridge();
891             unbridgedConfigServers[n] = newConn;
892         } else {
893             this._configServers[n] = newConn;
894         }
895 
896         this["config" + n] = this._configServers[n];
897         this["c" + n] = this._configServers[n];
898     };
899 
900     /**
901      * Helper method for setting primary shard of a database and making sure that it was successful.
902      * Note: first mongos needs to be up.
903      */
904     this.ensurePrimaryShard = function(dbName, shardName) {
905         var db = this.s0.getDB('admin');
906         var res = db.adminCommand({movePrimary: dbName, to: shardName});
907         assert(res.ok || res.errmsg == "it is already the primary", tojson(res));
908     };
909 
910     // ShardingTest initialization
911 
912     assert(isObject(params), 'ShardingTest configuration must be a JSON object');
913 
914     var testName = params.name || "test";
915     var otherParams = Object.merge(params, params.other || {});
916 
917     var numShards = otherParams.hasOwnProperty('shards') ? otherParams.shards : 2;
918     var mongosVerboseLevel = otherParams.hasOwnProperty('verbose') ? otherParams.verbose : 1;
919     var numMongos = otherParams.hasOwnProperty('mongos') ? otherParams.mongos : 1;
920     var numConfigs = otherParams.hasOwnProperty('config') ? otherParams.config : 3;
921     var waitForCSRSSecondaries = otherParams.hasOwnProperty('waitForCSRSSecondaries')
922         ? otherParams.waitForCSRSSecondaries
923         : true;
924 
925     // Allow specifying mixed-type options like this:
926     // { mongos : [ { noprealloc : "" } ],
927     //   config : [ { smallfiles : "" } ],
928     //   shards : { rs : true, d : true } }
929     if (Array.isArray(numShards)) {
930         for (var i = 0; i < numShards.length; i++) {
931             otherParams["d" + i] = numShards[i];
932         }
933 
934         numShards = numShards.length;
935     } else if (isObject(numShards)) {
936         var tempCount = 0;
937         for (var i in numShards) {
938             otherParams[i] = numShards[i];
939             tempCount++;
940         }
941 
942         numShards = tempCount;
943     }
944 
945     if (Array.isArray(numMongos)) {
946         for (var i = 0; i < numMongos.length; i++) {
947             otherParams["s" + i] = numMongos[i];
948         }
949 
950         numMongos = numMongos.length;
951     } else if (isObject(numMongos)) {
952         var tempCount = 0;
953         for (var i in numMongos) {
954             otherParams[i] = numMongos[i];
955             tempCount++;
956         }
957 
958         numMongos = tempCount;
959     }
960 
961     if (Array.isArray(numConfigs)) {
962         for (var i = 0; i < numConfigs.length; i++) {
963             otherParams["c" + i] = numConfigs[i];
964         }
965 
966         numConfigs = numConfigs.length;
967     } else if (isObject(numConfigs)) {
968         var tempCount = 0;
969         for (var i in numConfigs) {
970             otherParams[i] = numConfigs[i];
971             tempCount++;
972         }
973 
974         numConfigs = tempCount;
975     }
976 
977     otherParams.useHostname = otherParams.useHostname == undefined ? true : otherParams.useHostname;
978     otherParams.useBridge = otherParams.useBridge || false;
979     otherParams.bridgeOptions = otherParams.bridgeOptions || {};
980 
981     if (jsTestOptions().networkMessageCompressors) {
982         otherParams.bridgeOptions["networkMessageCompressors"] =
983             jsTestOptions().networkMessageCompressors;
984     }
985 
986     var keyFile = otherParams.keyFile;
987     var hostName = getHostName();
988 
989     this._testName = testName;
990     this._otherParams = otherParams;
991 
992     var pathOpts = {testName: testName};
993 
994     for (var k in otherParams) {
995         if (k.startsWith("rs") && otherParams[k] != undefined) {
996             break;
997         }
998     }
999 
1000     this._connections = [];
1001     this._rs = [];
1002     this._rsObjects = [];
1003 
1004     if (otherParams.useBridge) {
1005         var unbridgedConnections = [];
1006         var unbridgedConfigServers = [];
1007         var unbridgedMongos = [];
1008     }
1009 
1010     // Start the MongoD servers (shards)
1011     for (var i = 0; i < numShards; i++) {
1012         if (otherParams.rs || otherParams["rs" + i]) {
1013             var setName = testName + "-rs" + i;
1014 
1015             var rsDefaults = {
1016                 useHostname: otherParams.useHostname,
1017                 noJournalPrealloc: otherParams.nopreallocj,
1018                 oplogSize: 16,
1019                 shardsvr: '',
1020                 pathOpts: Object.merge(pathOpts, {shard: i}),
1021             };
1022 
1023             rsDefaults = Object.merge(rsDefaults, otherParams.rs);
1024             rsDefaults = Object.merge(rsDefaults, otherParams.rsOptions);
1025             rsDefaults = Object.merge(rsDefaults, otherParams["rs" + i]);
1026             rsDefaults.nodes = rsDefaults.nodes || otherParams.numReplicas;
1027             var rsSettings = rsDefaults.settings;
1028             delete rsDefaults.settings;
1029 
1030             var numReplicas = rsDefaults.nodes || 3;
1031             delete rsDefaults.nodes;
1032 
1033             var protocolVersion = rsDefaults.protocolVersion;
1034             delete rsDefaults.protocolVersion;
1035             var initiateTimeout = rsDefaults.initiateTimeout;
1036             delete rsDefaults.initiateTimeout;
1037 
1038             var rs = new ReplSetTest({
1039                 name: setName,
1040                 nodes: numReplicas,
1041                 useHostName: otherParams.useHostname,
1042                 useBridge: otherParams.useBridge,
1043                 bridgeOptions: otherParams.bridgeOptions,
1044                 keyFile: keyFile,
1045                 protocolVersion: protocolVersion,
1046                 settings: rsSettings
1047             });
1048 
1049             this._rs[i] =
1050                 {setName: setName, test: rs, nodes: rs.startSet(rsDefaults), url: rs.getURL()};
1051 
1052             rs.initiate(null, null, initiateTimeout);
1053 
1054             this["rs" + i] = rs;
1055             this._rsObjects[i] = rs;
1056 
1057             _alldbpaths.push(null);
1058             this._connections.push(null);
1059 
1060             if (otherParams.useBridge) {
1061                 unbridgedConnections.push(null);
1062             }
1063         } else {
1064             var options = {
1065                 useHostname: otherParams.useHostname,
1066                 noJournalPrealloc: otherParams.nopreallocj,
1067                 pathOpts: Object.merge(pathOpts, {shard: i}),
1068                 dbpath: "$testName$shard",
1069                 shardsvr: '',
1070                 keyFile: keyFile
1071             };
1072 
1073             if (jsTestOptions().shardMixedBinVersions) {
1074                 if (!otherParams.shardOptions) {
1075                     otherParams.shardOptions = {};
1076                 }
1077                 // If the test doesn't depend on specific shard binVersions, create a mixed version
1078                 // shard cluster that randomly assigns shard binVersions, half "latest" and half
1079                 // "last-stable".
1080                 if (!otherParams.shardOptions.binVersion) {
1081                     Random.setRandomSeed();
1082                     otherParams.shardOptions.binVersion =
1083                         MongoRunner.versionIterator(["latest", "last-stable"], true);
1084                 }
1085             }
1086 
1087             if (otherParams.shardOptions && otherParams.shardOptions.binVersion) {
1088                 otherParams.shardOptions.binVersion =
1089                     MongoRunner.versionIterator(otherParams.shardOptions.binVersion);
1090             }
1091 
1092             options = Object.merge(options, otherParams.shardOptions);
1093             options = Object.merge(options, otherParams["d" + i]);
1094 
1095             options.port = options.port || allocatePort();
1096 
1097             if (otherParams.useBridge) {
1098                 var bridgeOptions =
1099                     Object.merge(otherParams.bridgeOptions, options.bridgeOptions || {});
1100                 bridgeOptions = Object.merge(bridgeOptions, {
1101                     hostName: otherParams.useHostname ? hostName : "localhost",
1102                     // The mongod processes identify themselves to mongobridge as host:port, where
1103                     // the host is the actual hostname of the machine and not localhost.
1104                     dest: hostName + ":" + options.port,
1105                 });
1106 
1107                 var bridge = new MongoBridge(bridgeOptions);
1108             }
1109 
1110             var conn = MongoRunner.runMongod(options);
1111             if (!conn) {
1112                 throw new Error("Failed to start shard " + i);
1113             }
1114 
1115             if (otherParams.useBridge) {
1116                 bridge.connectToBridge();
1117                 this._connections.push(bridge);
1118                 unbridgedConnections.push(conn);
1119             } else {
1120                 this._connections.push(conn);
1121             }
1122 
1123             _alldbpaths.push(testName + i);
1124             this["shard" + i] = this._connections[i];
1125             this["d" + i] = this._connections[i];
1126 
1127             this._rs[i] = null;
1128             this._rsObjects[i] = null;
1129         }
1130     }
1131 
1132     // Do replication on replica sets if required
1133     for (var i = 0; i < numShards; i++) {
1134         if (!otherParams.rs && !otherParams["rs" + i]) {
1135             continue;
1136         }
1137 
1138         var rs = this._rs[i].test;
1139         rs.getPrimary().getDB("admin").foo.save({x: 1});
1140 
1141         if (keyFile) {
1142             authutil.asCluster(rs.nodes, keyFile, function() {
1143                 rs.awaitReplication();
1144             });
1145         }
1146 
1147         rs.awaitSecondaryNodes();
1148 
1149         var rsConn = new Mongo(rs.getURL());
1150         rsConn.name = rs.getURL();
1151 
1152         this._connections[i] = rsConn;
1153         this["shard" + i] = rsConn;
1154         rsConn.rs = rs;
1155     }
1156 
1157     this._configServers = [];
1158 
1159     // Using replica set for config servers
1160     var rstOptions = {
1161         useHostName: otherParams.useHostname,
1162         useBridge: otherParams.useBridge,
1163         bridgeOptions: otherParams.bridgeOptions,
1164         keyFile: keyFile,
1165         name: testName + "-configRS",
1166     };
1167 
1168     // when using CSRS, always use wiredTiger as the storage engine
1169     var startOptions = {
1170         pathOpts: pathOpts,
1171         // Ensure that journaling is always enabled for config servers.
1172         journal: "",
1173         configsvr: "",
1174         noJournalPrealloc: otherParams.nopreallocj,
1175         storageEngine: "wiredTiger",
1176     };
1177 
1178     if (otherParams.configOptions && otherParams.configOptions.binVersion) {
1179         otherParams.configOptions.binVersion =
1180             MongoRunner.versionIterator(otherParams.configOptions.binVersion);
1181     }
1182 
1183     startOptions = Object.merge(startOptions, otherParams.configOptions);
1184     rstOptions = Object.merge(rstOptions, otherParams.configReplSetTestOptions);
1185 
1186     var nodeOptions = [];
1187     for (var i = 0; i < numConfigs; ++i) {
1188         nodeOptions.push(otherParams["c" + i] || {});
1189     }
1190 
1191     rstOptions.nodes = nodeOptions;
1192 
1193     // Start the config server's replica set
1194     this.configRS = new ReplSetTest(rstOptions);
1195     this.configRS.startSet(startOptions);
1196 
1197     var config = this.configRS.getReplSetConfig();
1198     config.configsvr = true;
1199     config.settings = config.settings || {};
1200     var initiateTimeout = otherParams.rsOptions && otherParams.rsOptions.initiateTimeout;
1201     this.configRS.initiate(config, null, initiateTimeout);
1202 
1203     // Wait for master to be elected before starting mongos
1204     var csrsPrimary = this.configRS.getPrimary();
1205 
1206     // If chunkSize has been requested for this test, write the configuration
1207     if (otherParams.chunkSize) {
1208         function setChunkSize() {
1209             assert.writeOK(csrsPrimary.getDB('config').settings.update(
1210                 {_id: 'chunksize'},
1211                 {$set: {value: otherParams.chunkSize}},
1212                 {upsert: true, writeConcern: {w: 'majority', wtimeout: 30000}}));
1213         }
1214 
1215         if (keyFile) {
1216             authutil.asCluster(csrsPrimary, keyFile, setChunkSize);
1217         } else {
1218             setChunkSize();
1219         }
1220     }
1221 
1222     this._configDB = this.configRS.getURL();
1223     this._configServers = this.configRS.nodes;
1224     for (var i = 0; i < numConfigs; ++i) {
1225         var conn = this._configServers[i];
1226         this["config" + i] = conn;
1227         this["c" + i] = conn;
1228     }
1229 
1230     printjson('Config servers: ' + this._configDB);
1231 
1232     var configConnection = _connectWithRetry(this._configDB);
1233 
1234     print("ShardingTest " + this._testName + " :\n" +
1235           tojson({config: this._configDB, shards: this._connections}));
1236 
1237     this._mongos = [];
1238 
1239     // Start the MongoS servers
1240     for (var i = 0; i < numMongos; i++) {
1241         options = {
1242             useHostname: otherParams.useHostname,
1243             pathOpts: Object.merge(pathOpts, {mongos: i}),
1244             configdb: this._configDB,
1245             verbose: mongosVerboseLevel,
1246             keyFile: keyFile,
1247         };
1248 
1249         if (otherParams.mongosOptions && otherParams.mongosOptions.binVersion) {
1250             otherParams.mongosOptions.binVersion =
1251                 MongoRunner.versionIterator(otherParams.mongosOptions.binVersion);
1252         }
1253 
1254         options = Object.merge(options, otherParams.mongosOptions);
1255         options = Object.merge(options, otherParams["s" + i]);
1256 
1257         options.port = options.port || allocatePort();
1258 
1259         if (otherParams.useBridge) {
1260             var bridgeOptions =
1261                 Object.merge(otherParams.bridgeOptions, options.bridgeOptions || {});
1262             bridgeOptions = Object.merge(bridgeOptions, {
1263                 hostName: otherParams.useHostname ? hostName : "localhost",
1264                 // The mongos processes identify themselves to mongobridge as host:port, where the
1265                 // host is the actual hostname of the machine and not localhost.
1266                 dest: hostName + ":" + options.port,
1267             });
1268 
1269             var bridge = new MongoBridge(bridgeOptions);
1270         }
1271 
1272         var conn = MongoRunner.runMongos(options);
1273         if (!conn) {
1274             throw new Error("Failed to start mongos " + i);
1275         }
1276 
1277         if (otherParams.useBridge) {
1278             bridge.connectToBridge();
1279             this._mongos.push(bridge);
1280             unbridgedMongos.push(conn);
1281         } else {
1282             this._mongos.push(conn);
1283         }
1284 
1285         if (i === 0) {
1286             this.s = this._mongos[i];
1287             this.admin = this._mongos[i].getDB('admin');
1288             this.config = this._mongos[i].getDB('config');
1289         }
1290 
1291         this["s" + i] = this._mongos[i];
1292     }
1293 
1294     _extendWithShMethods();
1295 
1296     // If auth is enabled for the test, login the mongos connections as system in order to configure
1297     // the instances and then log them out again.
1298     if (keyFile) {
1299         authutil.asCluster(this._mongos, keyFile, _configureCluster);
1300     } else {
1301         _configureCluster();
1302     }
1303 
1304     try {
1305         if (!otherParams.manualAddShard) {
1306             this._shardNames = [];
1307 
1308             var testName = this._testName;
1309             var admin = this.admin;
1310             var shardNames = this._shardNames;
1311 
1312             this._connections.forEach(function(z) {
1313                 var n = z.name || z.host || z;
1314 
1315                 print("ShardingTest " + testName + " going to add shard : " + n);
1316 
1317                 var result = admin.runCommand({addshard: n});
1318                 assert.commandWorked(result, "Failed to add shard " + n);
1319 
1320                 shardNames.push(result.shardAdded);
1321                 z.shardName = result.shardAdded;
1322             });
1323         }
1324     } catch (e) {
1325         // Clean up the running procceses on failure
1326         print("Failed to add shards, stopping cluster.");
1327         this.stop();
1328         throw e;
1329     }
1330 
1331     if (waitForCSRSSecondaries) {
1332         // Ensure that all CSRS nodes are up to date. This is strictly needed for tests that use
1333         // multiple mongoses. In those cases, the first mongos initializes the contents of the
1334         // 'config' database, but without waiting for those writes to replicate to all the
1335         // config servers then the secondary mongoses risk reading from a stale config server
1336         // and seeing an empty config database.
1337         this.configRS.awaitLastOpCommitted();
1338     }
1339 
1340     if (jsTestOptions().keyFile) {
1341         jsTest.authenticate(configConnection);
1342         jsTest.authenticateNodes(this._configServers);
1343         jsTest.authenticateNodes(this._mongos);
1344     }
1345 };
1346