Dynamic database connection to mongodb or mongoose from nodejs
This is to help others who may find themselves in similar situation as I did. I hope that it could be standardized. I dont think we should have to reinvent the wheel every time someone needs to make a multi-tenant application.
This example describes a multi-tenant structure with each client having its own database. Like i said there might be a better way of doing this, but because i didn't get help myself, this was my solution.
So here are the goals this solution targets:
- each client is identified by subdomain e.g client1.application.com,
- application checks if subdomain is valid,
- application looks up and obtains connection information (database url, credentials, etc) from master database,
- application connects to client database ( pretty much hands over to client),
- application takes measures to ensure integrity and resource management (e.g use the same database connection for members of the same client, rather than make new connection).
Here is the code
in your app.js file
app.use(clientListener()); // checks and identify valid clients
app.use(setclientdb());// sets db for valid clients
I've created two middlewares :
clientListener- to identify the client connecting,setclientdb- gets client details from Master database, after client is identified, and then establishes connection to client database.
clientListener middleware
I check who the client is by checking the subdomain from the request object. I do a bunch of checks to be sure the client is valid (I know the code is messy, and can be made cleaner). After ensuring the client is valid, I store the clients info in session. I also check that if the clients info is already stored in session, there is no need to query the database again. We just need to make sure that the request subdomain, matches that which is already stored in session.
var Clients = require('../models/clients');
var basedomain = dbConfig.baseDomain;
var allowedSubs = {'admin':true, 'www':true };
allowedSubs[basedomain] = true;
function clientlistener() {
return function(req, res, next) {
//console.dir('look at my sub domain ' + req.subdomains[0]);
// console.log(req.session.Client.name);
if( req.subdomains[0] in allowedSubs || typeof req.subdomains[0] === 'undefined' || req.session.Client && req.session.Client.name === req.subdomains[0] ){
//console.dir('look at the sub domain ' + req.subdomains[0]);
//console.dir('testing Session ' + req.session.Client);
console.log('did not search database for '+ req.subdomains[0]);
//console.log(JSON.stringify(req.session.Client, null, 4));
next();
}
else{
Clients.findOne({subdomain: req.subdomains[0]}, function (err, client) {
if(!err){
if(!client){
//res.send(client);
res.send(403, 'Sorry! you cant see that.');
}
else{
console.log('searched database for '+ req.subdomains[0]);
//console.log(JSON.stringify(client, null, 4));
//console.log(client);
// req.session.tester = "moyo cow";
req.session.Client = client;
return next();
}
}
else{
console.log(err);
return next(err)
}
});
}
}
}
module.exports = clientlistener;
setclientdb middleware:
I check everything again making sure that the client is valid. Then the connection to the client's database with the info retrieved from session is opened.
I also make sure to store all active connections into a global object, so as to prevent new connections to the database upon each request(we don't want to overload each clients mongodb server with connections).
var mongoose = require('mongoose');
//var dynamicConnection = require('../models/dynamicMongoose');
function setclientdb() {
return function(req, res, next){
//check if client has an existing db connection /*** Check if client db is connected and pooled *****/
if(/*typeof global.App.clientdbconn === 'undefined' && */ typeof(req.session.Client) !== 'undefined' && global.App.clients[req.session.Client.name] !== req.subdomains[0])
{
//check if client session, matches current client if it matches, establish new connection for client
if(req.session.Client && req.session.Client.name === req.subdomains[0] )
{
console.log('setting db for client ' + req.subdomains[0]+ ' and '+ req.session.Client.dbUrl);
client = mongoose.createConnection(req.session.Client.dbUrl /*, dbconfigoptions*/);
client.on('connected', function () {
console.log('Mongoose default connection open to ' + req.session.Client.name);
});
// When the connection is disconnected
client.on('disconnected', function () {
console.log('Mongoose '+ req.session.Client.name +' connection disconnected');
});
// If the Node process ends, close the Mongoose connection
process.on('SIGINT', function() {
client.close(function () {
console.log(req.session.Client.name +' connection disconnected through app termination');
process.exit(0);
});
});
//If pool has not been created, create it and Add new connection to the pool and set it as active connection
if(typeof(global.App.clients) === 'undefined' || typeof(global.App.clients[req.session.Client.name]) === 'undefined' && typeof(global.App.clientdbconn[req.session.Client.name]) === 'undefined')
{
clientname = req.session.Client.name;
global.App.clients[clientname] = req.session.Client.name;// Store name of client in the global clients array
activedb = global.App.clientdbconn[clientname] = client; //Store connection in the global connection array
console.log('I am now in the list of active clients ' + global.App.clients[clientname]);
}
global.App.activdb = activedb;
console.log('client connection established, and saved ' + req.session.Client.name);
next();
}
//if current client, does not match session client, then do not establish connection
else
{
delete req.session.Client;
client = false;
next();
}
}
else
{
if(typeof(req.session.Client) === 'undefined')
{
next();
}
//if client already has a connection make it active
else{
global.App.activdb = global.App.clientdbconn[req.session.Client.name];
console.log('did not make new connection for ' + req.session.Client.name);
return next();
}
}
}
}
module.exports = setclientdb;
Last but not the least
Since I am using a combination of mongoose and native mongo, We have to compile our models at run time. Please see below
Add this to your app.js
// require your models directory
var models = require('./models');
// Create models using mongoose connection for use in controllers
app.use(function db(req, res, next) {
req.db = {
User: global.App.activdb.model('User', models.agency_user, 'users')
//Post: global.App.activdb.model('Post', models.Post, 'posts')
};
return next();
});
Explanation:
Like I said earlier on I created a global object to store the active database connection object: global.App.activdb
Then I use this connection object to create (compile) mongoose model, after i store it in the db property of the req object: req.db. I do this so that i can access my models in my controller like this for example.
Example of my Users controller:
exports.list = function (req, res) {
req.db.User.find(function (err, users) {
res.send("respond with a resource" + users + 'and connections ' + JSON.stringify(global.App.clients, null, 4));
console.log('Worker ' + cluster.worker.id + ' running!');
});
};
I will come back and clean this up eventually. If anyone wants to help me, that be nice.
Hello everyone, here is a much more updated solution.
So here are the goals this solution targets:
- each client is identified by subdomain e.g client1.application.com,
- application checks if subdomain is valid,
- application looks up and obtains connection information (database url, credentials, etc) from master database,
- application connects to client database ( pretty much hands over to client),
- application takes measures to ensure integrity and resource management (e.g use the same database connection for members of the same client, rather than make new connection).
updates
- use of promises,
- automatic import & compilation of models
- New middleware ; modelsinit (used to automatically import and compile mongoose models)
- Clean up of middlewares (setclientdb, clientlistener, modelsInit)
Please see below for some Explanations
**
modelsInit Middleware
** features
- tests if models are already compiled. If so, skip.
tests to see if request is not a tenant request; i.e (request to apps homepage, admin page, etc)
'use strict'; /** * Created by moyofalaye on 3/17/14. */ var path = require('path'); var config = require('../../config/config'); // Globbing model files config.getGlobbedFiles('./app/models/*.js').forEach(function (modelPath) { require(path.resolve(modelPath)); }); function modelsInit() { return function (req, res, next) { //console.log(req.subdomains[0]); switch (req.subdomains[0]) { case 'www': case undefined: return next(); break; case 'admin': return next(); break; // default: // return } var clientname = req.session.Client.name; // test if models are not already compiled if so, skip if (/*typeof req.db === 'undefined' && */ typeof global.App.clientModel[clientname] === 'undefined') { req.db = {}; //Get files from models directory config.getGlobbedFiles('./app/models/clientmodels/**/*.js').forEach(function (modelPath) { console.log('the filepath is ' + modelPath); //Deduce/ extrapulate model names from the file names //Im not very good with regxp but this is what i had to do, to get the names from the filename e.g users.server.models.js (this is my naming convention, so as not to get confused with server side models and client side models var filename = modelPath.replace(/^.*[\\\/]/, ''); var fullname = filename.substr(0, filename.lastIndexOf('.')); var endname = fullname.indexOf('.'); var name = fullname.substr(0, endname); req.db[name] = require(path.resolve(modelPath))(global.App.activdb); console.log('the filename is ' + name); }); global.App.clientModel[clientname] = req.db; console.log(global.App.clients); return next(); } // since models exist, pass it to request.db for easy consumption in controllers req.db = global.App.clientModel[clientname]; return next(); }; } module.exports = modelsInit;
Todo: Further Explanation
ClientListener.js
var config = require('../../config/config');
var Clients = require('../models/clients');
var basedomain = config.baseDomain;
var allowedSubs = {'admin': true, 'www': true};
allowedSubs[basedomain] = true;
//console.dir(allowedSubs);
function clientlistener() {
return function (req, res, next) {
//check if client has already been recognized
if (req.subdomains[0] in allowedSubs || typeof req.subdomains[0] == 'undefined' || req.session.Client && req.session.Client.name === req.subdomains[0]) {
console.log('did not search database for ' + req.subdomains[0]);
//console.log(JSON.stringify(req.session.Client, null, 4));
return next();
}
//look for client in database
else {
Clients.findOne({subdomain: req.subdomains[0]}, function (err, client) {
if (!err) {
//if client not found
if (!client) {
//res.send(client);
res.status(403).send('Sorry! you cant see that.');
console.log(client);
}
// client found, create session and add client
else {
console.log('searched database for ' + req.subdomains[0]);
req.session.Client = client;
return next();
}
}
else {
console.log(err);
return next(err)
}
});
}
}
}
module.exports = clientlistener;
setclientdb.js
var client;
var clientname;
var activedb;
var Promise = require("bluebird");
Promise.promisifyAll(require("mongoose"));
//mongoose = require('mongoose');
function setclientdb() {
return function (req, res, next) {
//check if client is not valid
if (typeof(req.session.Client) === 'undefined' || req.session.Client && req.session.Client.name !== req.subdomains[0]) {
delete req.session.Client;
client = false;
return next();
}
//if client already has an existing connection make it active
else if (global.App.clients.indexOf(req.session.Client.name) > -1) {
global.App.activdb = global.App.clientdbconn[req.session.Client.name]; //global.App.clientdbconnection is an array of or established connections
console.log('did not make new connection for ' + req.session.Client.name);
return next();
}
//make new db connection
else {
console.log('setting db for client ' + req.subdomains[0] + ' and ' + req.session.Client.dbUrl);
client = mongoose.createConnection(req.session.Client.dbUrl /*, dbconfigoptions*/);
client.on('connected', function () {
console.log('Mongoose default connection open to ' + req.session.Client.name);
//If pool has not been created, create it and Add new connection to the pool and set it as active connection
if (typeof(global.App.clients) === 'undefined' || typeof(global.App.clients[req.session.Client.name]) === 'undefined' && typeof(global.App.clientdbconn[req.session.Client.name]) === 'undefined') {
clientname = req.session.Client.name;
global.App.clients.push(req.session.Client.name);// Store name of client in the global clients array
activedb = global.App.clientdbconn[clientname] = client; //Store connection in the global connection array and set it as the current active database
console.log('I am now in the list of active clients ' + global.App.clients[clientname]);
global.App.activdb = activedb;
console.log('client connection established, and saved ' + req.session.Client.name);
return next();
}
});
// When the connection is disconnected
client.on('disconnected', function () {
console.log('Mongoose ' + req.session.Client.name + ' connection disconnected');
});
// If the Node process ends, close the Mongoose connection
process.on('SIGINT', function () {
client.close(function () {
console.log(req.session.Client.name + ' connection disconnected through app termination');
process.exit(0);
});
});
}
}
}
module.exports = setclientdb;