WAR like application deployment in node.js

Taking on interesting ideas is one of my favorite hobbies. While doing a bit of Youtube browsing yesterday, I came across a talk by James A. Gosling, the father of Java on scalability:

Hmm, an interesting prospect. What if memory was more abundant, and the concern was direct access to objects in said memory? The idea here being to minimize file IO. Contemplating on the way Java operates, I become intrigued on the idea of loading an entire application into memory, much as the way various Java based servers load up war files.

To start out with this crazy idea, I decided to utilize a libzip based npm module, zipfile. It basically loads a single zip file, and gives a names property with an array of file names. The end result looks something like this:

> var zipfile = require('zipfile').ZipFile;
> var zf = new zipfile('site.zip');
> console.log(zf.names);
[ 'root/',
  'root/index.html',
...

Next, I thought up an idea of turning this into a JavaScript object, where paths and files are object properties, and files point to the actual file contents. In this way we can “load the site into memory” so to speak. Here’s the bit of code that accomplishes this:

Note: You will need the zipfile npm module for this. npm install zipfile will get you going

var ziptree = {};

function isDir(pathname) {
 if(!pathname)
 {
   return false;
 }
 else {
   return pathname.slice(-1) == '/';
 }
}

function createDirectoryStructure(pathname) {
  var split_pathname = pathname.split('/');
  var iterator = ziptree;

  // Create all paths, so we want the length - 1
  var i;
  for(i = 0; i < (split_pathname.length - 1); i++) {
    if(!iterator.hasOwnProperty(split_pathname[i])) {
      iterator[split_pathname[i]] = {};
    }

    iterator = iterator[split_pathname[i]];
  }

  iterator[split_pathname[i]] = site_zip.readFileSync(pathname);
}

var zipfile = require('zipfile').ZipFile;
var util = require('util');
var site_zip = new zipfile('./site.zip');

for(var i = 0; i < site_zip.names.length ; i++) {
  if(!isDir(site_zip.names[i])) {
    createDirectoryStructure(site_zip.names[i]);
  }
}

The use of iterator let’s us do something of a chdir for the process of creating recursive directory structure. Fortunately this only occurs if the said property doesn’t exist already. Unfortunately this requires a blocking readFileSync at the moment, since async calls appear to cause the dreaded too many file handles open issue. Then again, the idea is to read in all of our zip file archives before the server even starts, so we’re not blocking client requests at the moment. It does, however, mean that the server won’t be able to accept clients right away until all the appropriate deployment files are loaded. Here is a simple example of sending back a response to a client using this new object:

// Quick helper function
function isRoot() { return process.getuid() == 0; }

var process_user = '';
var port_number = 80;

// A single argument was passed in
if(process.argv.length == 3) {
  // Check if this is the port number or user to bind as
  if(parseInt(process.argv[2])) {
    port_number = process.argv[2];
  }
  else {
    process_user = process.argv[2];
  }
}
// port and user passed in
else if(process.argv.length == 4) {
  if(!parseInt(process.argv[2])) {
    console.error("Invalid port number: '%s'", process.argv[2]);
    process.exit(1);
  }

  process_user = process.argv[3];
}

if(port_number < 1024 && !isRoot()) {
  console.error("Binding to ports less than 1024 requires root privileges.")
  process.exit(1);
}
// Check to see if we're trying to run as root
// with no privileged user set.
else if (isRoot() && !process_user) {
  console.error("Please provide a non-privileged user to bind as.")
  console.error("Usage: node restart_server.js [port] user");
  process.exit(1);
}

var http = require('http');
var server;

function StartServer() {
  console.log("Starting server [%d]...", process.pid);
  // Initalizations such as reading the config file, etc.
  server = http.createServer(function (req, res) {
  res.writeHead(200, {'Content-Type': 'text/html'});
  res.end(ziptree['root']['index.html']);
  });
  try {
    server.listen(port_number, "::1");
    if(process_user) {
      process.setuid(process_user);
    }
  }
  catch(err) {
    console.error("Error: [%s] Call: [%s]", err.message, err.syscall);
    process.exit(1);
  }
}

function StopServer(shutdown) {
  console.log("Stopping server...");
  // Other cleanup functions here
  if(server) {
    server.close();
    if(shutdown) {
      console.log("Exiting server process.");
      process.exit();
    }
  }
}

process.on("SIGHUP", function(){
  console.log("Received SIGHUP, restarting server...");
  StopServer();
  StartServer();
});

process.on("SIGINT", function(){
  console.log("Received SIGINT, cleaning up...");
  StopServer(true);
});

process.on("SIGTERM", function(){
  console.log("Received SIGTERM, cleaning up...");
  StopServer(true);
});

StartServer();

This sets up a server using the code from my last post on process.setuid servers. The main point of interest is here:

  server = http.createServer(function (req, res) {
  res.writeHead(200, {'Content-Type': 'text/html'});
  res.end(ziptree['root']['index.html']);
  });

Where as we see the buffer object held in ziptree['root']['index.html'] is sent to the client. We can confirm this works by doing a quick CURL call:

$ curl http://localhost/
<html>
<head>
</head>
<body>
<h1>Index Page</h1>
<p>This is my sweet website</p>
</body>
</html>

There you have it, no filesystem operations involved at all. Now a few more points I’d like to work on for this system:

• An application configuration in the zip file, something like a deploy.json that handles routing and other configurations
• Handling dynamic code, most likely using VM Context
• Using timestamps on the loaded files so we can check modification times to see if files need to be reloaded. Can we literally do hot updates of code?
• Using POSIX Signals to indicate that new archive files exist and need to be loaded. This will prevent the need for interval checks for new files, which hit the filesystem. We’re trying to reduce that!
• Whatever else I can’t think of at the moment

I’ll see how this goes, but if nothing else it’s an interesting concept to play with.