Replica sets use elections to determine which set member will become primary. Replica sets can trigger an election in response to a variety of events, such as:
Adding a new node to the replica set,
initiating a replica set,
performing replica set maintenance using methods such as rs.stepDown() or rs.reconfig(), and
the secondary members losing connectivity to the primary for more than the configured timeout (10 seconds by default).
In the following diagram, the primary node was unavailable for longer than the configured timeout and triggers the automatic failover process. One of the remaining secondaries calls for an election to select a new primary and automatically resume normal operations.
1. Login into primary
[[email protected] ~]$ mongo --port 27017
MongoDB shell version: 2.6.3
connecting to: 127.0.0.1:27017/test
myreplicaset:PRIMARY>
2. Store configuration to a variable “config”
myreplicaset:PRIMARY> config=rs.conf();
{
"_id" : "myreplicaset",
"version" : 3,
"members" : [
{
"_id" : 0,
"host" : "localhost.localdomain:27017"
},
{
"_id" : 1,
"host" : "localhost.localdomain:27018"
},
{
"_id" : 2,
"host" : "localhost.localdomain:27019"
}
]
}
3. Update priority of members myreplicaset:PRIMARY> config.members[0].priority=3
myreplicaset:PRIMARY> config.members[1].priority=1 1
myreplicaset:PRIMARY> config.members[2].priority=2 2
4. Reconfigure ReplicaSet with new configurations myreplicaset:PRIMARY>
rs.reconfig(config)
2014-07-08T15:08:24.009+0200 DBClientCursor::init call() failed
2014-07-08T15:08:24.012+0200 trying reconnect to 127.0.0.1:27017
(127.0.0.1) failed
2014-07-08T15:08:24.013+0200 reconnect 127.0.0.1:27017 (127.0.0.1) ok reconnected to server after rs command (which is normal)
5. View ReplicaSet configuration myreplicaset:PRIMARY> rs.conf();
{
"_id" : "myreplicaset",
"version" : 4,
"members" : [
{
"_id" : 0,
"host" : "localhost.localdomain:27017",
"priority" : 3
},
{
"_id" : 1,
"host" : "localhost.localdomain:27018"
},
{
"_id" : 2,
"host" : "localhost.localdomain:27019",
"priority" : 2
}
]
}
6. Check status of the ReplicaSet myreplicaset:PRIMARY> rs.status();
{
"set" : "myreplicaset",
"date" : ISODate("2014-07-08T13:09:19Z"),
"myState" : 1,
"members" : [
{
"_id" : 0,
"name" : "localhost.localdomain:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 1134,
"optime" : Timestamp(1404824903, 1),
"optimeDate" : ISODate("2014-07-08T13:08:23Z"),
"electionTime" : Timestamp(1404824025, 1),
"electionDate" : ISODate("2014-07-08T12:53:45Z"),
"self" : true
},
{
"_id" : 1,
"name" : "localhost.localdomain:27018",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 56,
"optime" : Timestamp(1404824903, 1),
"optimeDate" : ISODate("2014-07-08T13:08:23Z"),
"lastHeartbeat" : ISODate("2014-07-08T13:09:17Z"),
"lastHeartbeatRecv" : ISODate("2014-07-08T13:09:18Z"),
"pingMs" : 0,
"lastHeartbeatMessage" : "syncing to:
localhost.localdomain:27017",
"syncingTo" : "localhost.localdomain:27017"
},
{
"_id" : 2,
"name" : "localhost.localdomain:27019",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 56,
"optime" : Timestamp(1404824903, 1),
"optimeDate" : ISODate("2014-07-08T13:08:23Z"),
"lastHeartbeat" : ISODate("2014-07-08T13:09:17Z"),
"lastHeartbeatRecv" : ISODate("2014-07-08T13:09:19Z"),
"pingMs" : 0,
"lastHeartbeatMessage" : "syncing to:
localhost.localdomain:27017",
"syncingTo" : "localhost.localdomain:27017"
}
],
"ok" : 1
}
7. Exit from the primary
myreplicaset:PRIMARY> exit
bye
8. Stop the current master process
[[email protected] ~]$ sudo su [sudo] password for tom: [[email protected] tom]# cd /var/lib/mongo [[email protected] mongo]# ps -ef|grep mongo
root 2984 1 0 14:50 ? 00:00:03 mongod --port 27017 -
dbpath /var/lib/mongo/replicaset0 --logpath
/var/lib/mongo/replicaset0/log --replSet myreplicaset --fork --
smallfiles --nojournal
root 3036 1 0 14:51 ? 00:00:02 mongod --port 27018 -
dbpath /var/lib/mongo/replicaset1 --logpath
/var/lib/mongo/replicaset1/log --replSet myreplicaset --fork --
smallfiles --nojournal
root 3082 1 0 14:51 ? 00:00:02 mongod --port 27019 -
dbpath /var/lib/mongo/replicaset2 --logpath /var/lib/mongo/replicaset2/log --replSet myreplicaset --fork -- smallfiles --nojournal
root 3699 3682 0 15:11 pts/2 00:00:00 grep mongo
[[email protected] mongo]# kill -9 2984
9. Login into mongo with priority 2
[[email protected] mongo]# mongo --port 27019
MongoDB shell version: 2.6.3
connecting to: 127.0.0.1:27019/test
myreplicaset:PRIMARY>
10. Check configuration myreplicaset:PRIMARY> rs.conf();
{
"_id" : "myreplicaset",
"version" : 4,
"members" : [
{
"_id" : 0,
"host" : "localhost.localdomain:27017",
"priority" : 3
},
{
"_id" : 1,
"host" : "localhost.localdomain:27018"
},
{
"_id" : 2,
"host" : "localhost.localdomain:27019",
"priority" : 2
}
]
}
11. Check ReplicaSet status myreplicaset:PRIMARY> rs.status();
{
"set" : "myreplicaset",
"date" : ISODate("2014-07-08T13:13:37Z"),
"myState" : 1,
"members" : [
{
"_id" : 0,
"name" : "localhost.localdomain:27017",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(1404824903, 1),
"optimeDate" : ISODate("2014-07-08T13:08:23Z"),
"lastHeartbeat" : ISODate("2014-07-08T13:13:35Z"),
"lastHeartbeatRecv" : ISODate("2014-07-08T13:12:12Z"),
"pingMs" : 0
},
{
"_id" : 1,
"name" : "localhost.localdomain:27018",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 312,
"optime" : Timestamp(1404824903, 1),
"optimeDate" : ISODate("2014-07-08T13:08:23Z"),
"lastHeartbeat" : ISODate("2014-07-08T13:13:35Z"),
"lastHeartbeatRecv" : ISODate("2014-07-08T13:13:36Z"),
"pingMs" : 0,
"lastHeartbeatMessage" : "syncing to:
localhost.localdomain:27019",
"syncingTo" : "localhost.localdomain:27019"
},
{
"_id" : 2,
"name" : "localhost.localdomain:27019",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 1325,
"optime" : Timestamp(1404824903, 1),
"optimeDate" : ISODate("2014-07-08T13:08:23Z"),
"electionTime" : Timestamp(1404825141, 1),
"electionDate" : ISODate("2014-07-08T13:12:21Z"),
"self" : true
}
],
"ok" : 1
}
12. Exit from the current primary
myreplicaset:PRIMARY> exit
bye
13. Restart the killed process [[email protected] mongo]# rm replicaset0/mongod.lock
rm: remove regular file `replicaset0/mongod.lock'? yes
[[email protected] mongo]# mongod --port 27017 --dbpath /var/lib/mongo/replicaset0 --logpath /var/lib/mongo/replicaset0/log -- replSet myreplicaset --fork --smallfiles --nojournal
about to fork child process, waiting until server is ready for connections.
forked process: 5293
child process started successfully, parent exiting
14. Login into the restarted process [[email protected] mongo]# mongo --port 27017
MongoDB shell version: 2.6.3
connecting to: 127.0.0.1:27017/test
15. Check ReplicaSet configuration myreplicaset:PRIMARY> rs.conf();
{
"_id" : "myreplicaset",
"version" : 4,
"members" : [
{
"_id" : 0,
"host" : "localhost.localdomain:27017",
"priority" : 3
},
{
"_id" : 1,
"host" : "localhost.localdomain:27018"
},
{
"_id" : 2,
"host" : "localhost.localdomain:27019",
"priority" : 2
}
]
}
16. Check ReplicaSet status myreplicaset:PRIMARY> rs.status();
{
"set" : "myreplicaset",
"date" : ISODate("2014-07-08T13:17:33Z"),
"myState" : 1,
"members" : [
{
"_id" : 0,
"name" : "localhost.localdomain:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 39,
"optime" : Timestamp(1404824903, 1),
"optimeDate" : ISODate("2014-07-08T13:08:23Z"),
"electionTime" : Timestamp(1404825417, 1),
"electionDate" : ISODate("2014-07-08T13:16:57Z"),
"self" : true
},
{
"_id" : 1,
"name" : "localhost.localdomain:27018",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 38,
"optime" : Timestamp(1404824903, 1),
"optimeDate" : ISODate("2014-07-08T13:08:23Z"),
"lastHeartbeat" : ISODate("2014-07-08T13:17:33Z"),
"lastHeartbeatRecv" : ISODate("2014-07-08T13:17:33Z"),
"pingMs" : 0,
"lastHeartbeatMessage" : "syncing to:
localhost.localdomain:27019",
"syncingTo" : "localhost.localdomain:27019"
},
{
"_id" : 2,
"name" : "localhost.localdomain:27019",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 38,
"optime" : Timestamp(1404824903, 1),
"optimeDate" : ISODate("2014-07-08T13:08:23Z"),
"lastHeartbeat" : ISODate("2014-07-08T13:17:33Z"),
"lastHeartbeatRecv" : ISODate("2014-07-08T13:17:32Z"),
"pingMs" : 0,
"lastHeartbeatMessage" : "syncing to:
localhost.localdomain:27017",
"syncingTo" : "localhost.localdomain:27017"
}
],
"ok" : 1
}
17. Exit from the primary.
myreplicaset:PRIMARY> exit
bye