Posts in June 2013 - Jason Connell

Node.js Keeps Me Sane June 26, 2013

Scenario: We're rewriting the code for a site but it's pretty much using the same javascript and css currently. So I have a copy of the old version running and a copy of the new version. And something is not perfect in the new site. A Javascript problem. To eliminate differences in javascript as a cause of the problem, here's what I did:

var http = require("http"),
	fs = require("fs"),
	SyncArray = require("syncarray").SyncArray,
	fshelp = require("filesystem");
require("strings");

var web1 = "http://localhost/";
var web2 = "http://oldsite.com/";

var name1 = "new";
var name2 = "old";


function processRequest(baseUrl, url, name, baseDir, finished){
	fshelp.ensureFolder(baseDir, function(){
		http.get(baseUrl + url, function(resp){
			var file = url.lastIndexOf("/") == url.length - 1 ? "_index.html" : url.substring(url.lastIndexOf("/"));
			resp.setEncoding("utf8");

			var data = "";
			resp.on("data", function(d){
				data += d;
			});

			resp.on("end", function(){
				fs.writeFile(baseDir + "\\" + file, data, "utf8", function(err){
					if (err) console.log("error encountered writing file - " + JSON.stringify(err));
					
					var files = findUrls(data);
					console.log(files.length);
					var sync = new SyncArray(files);
					sync.forEach(function(f, i, arr, finishedOne){
						console.log("processing " + f);
						processRequest(baseUrl, f, name, baseDir, function(){
							finishedOne();
						});
					}, function(){
						finished();
					})
				});

			});

		});
	});
}

function findUrls(content){
	var files = [];
	var reg = /(src|href)=["|']([a-zA-Z0-9\._\-\/]*?)["|']/g;

	var m = null;
	while ((m = reg.exec(content)) != null){
		if (m[2].trim().length > 0 && m[2].indexOf(".js") != -1){
			files.push(m[2]);
		}
	}
	return files;
}



processRequest(web1, "", name1, "c:\\webs\\" + name1, function(){
	console.log("Done " + web1);
});

processRequest(web2, "", name2, "c:\\webs\\" + name2, function(){
	console.log("Done " + web2);
});

After that was complete, I then used WinMerge. It would find all javascript files in the html of the home page (what we were having trouble with), and save them all to a folder. It turned out they were identical. But part of narrowing in on the problem is eliminating other possible causes. And I found the problem in the HTML, in where some JS files were being included. Some in head, some below. Putting them all below fixed everything.

I write Node.js in work too. June 21, 2013

Read my blog post on Node.js as a tool to get work done...

Credit Card Payment Plan June 14, 2013

I figured out this little trick. Now that banks offer automatic payments to credit cards, there's a pretty persistent way to make a dent.

Say your bill is $4,000 and your minimum payment is $150. Instead of paying $150 on the due date (a pretty sizable chunk to be missing during one pay period), do $75 every other week. Schedule it in a way that the 2nd payment is a day or two before the due date, so that they can't hit you with a late fee, as scumbag banks like to do.

My credit card bill just hit below $2,000. I've actually been paying $100 every two weeks, so I started it approximately 9 months ago. It's pretty nice. The problem is I have two other credit cards with outstanding balances!! I'm getting them down there using the same strategy. Give it a try, especially if you can auto-deduct it, then you don't even have to worry about remembering to do it!

I'm an IDIOT!! June 11, 2013

I spent a ton of time trying to write a syntax parser for a meta-language which would be parsed by Javascript. I just replaced a lot of confusing logic with about 40 lines of code.

First, Javascript is dynamic and has stuff built in that can evaluate arbitrary code passed in via a string.

I'm not talking about eval().

Function!

However, my syntax looks like this: Hello {{page.user == null ? 'Anonymous' : page.user.firstName }}

So, we need to know what "page" is in the context of the function. I still have to parse out top level variables. In the code above, it will get "page" as a top level variable.

Then, I build up the function:

var f = new Function(vars[0], body);

"body" is actually modified, I set it to "return " + body; So that I can do {{ page.user == null ? 'Anonymous' : page.user.firstName }} and it will return the display name instead of undefined, the default behavior of a void function.

I have to count up the number of variables used, and build the function accordingly. Currently, this is a switch statement.

switch (vars.length){

case 0: f = new Function(body); break;

case 1: f = new Function(vars[0], body); break;

case 2: f = new Function(vars[0], vars[1], body); break;

}

Luckily in my code, there aren't more than 3-4 "top level" variables, including globals like "Array" and "String".

Here's the variable parsing part:

var shared = require("./shared");
require("strings");

var constants = { "true": true, "false": true, "null": true };

var lookup = {};

this.getVariables = function(body){
	if (body in lookup)
		return lookup[body];

	var vars = [];
	var instr = false;
	var instrch = null;
	var buf = "";
	var toplevel = false;
	var result = null;

	for (var i = 0; i < body.length; i++){
		var ch = body.charAt(i);

		switch (ch){
			case "'": case "\"":
				instr = ch != instrch;
				instrch = instrch == null ? ch : (instr ? instrch : null);
				break;
		}

		if ((!instr && shared.tokenSeparator.test(ch)) || i == body.length-1){
			if (i == body.length-1) buf+= ch;
			if (!toplevel && (result = shared.variable.exec(buf)) != null && !(result[1] in constants)){
				if (!vars.some(function(d){ return d == result[1]})){
					vars.push(result[1]);
					toplevel = true;
				}
			}
			buf = "";
		}
		else if (instr) buf = "";
		else buf += ch;

		if (toplevel && (instr || (shared.tokenSeparator.test(ch) && ch != "." && ch != "]"))) toplevel = false;
	}

	lookup[body] = vars;
	return vars;
}

And here's the evaluation part:

var syntax = require("./syntax");
var shared = require("./shared");

var lookup = {};

var Evaluator = function(globals){
	this.globals = globals;
}

Evaluator.prototype.evaluate = function(body, context){
	body = shared.replaceComps(body);

	var vars = syntax.getVariables(body);
	var args = [];

	body = "return " + body;

	for (var i = 0; i < vars.length; i++){
		if (context && vars[i] in context)
			args.push(context[vars[i]]);
		else if (this.globals && vars[i] in this.globals)
			args.push(this.globals[vars[i]]);
	}

	if (body in lookup){
		return lookup[body].apply(null, args);
	}

	var f = null;
	switch (vars.length){
		case 0: f = new Function(body); break;
		case 1: f = new Function(vars[0], body); break;
		case 2: f = new Function(vars[0], vars[1], body); break;
		case 3: f = new Function(vars[0], vars[1], vars[2], body); break;
		case 4: f = new Function(vars[0], vars[1], vars[2], vars[3], body); break;
	}

	var result = null;
	if (f != null){
		result = f.apply(null, args);
		lookup[body] = f;
	} 

	return result;
}

this.Evaluator = Evaluator;

shared.js has a regular expression, a map (for old syntax considerations), and a function to replace some old ways of doing things with the new, pure Javascript way of doing it.

this.replCompRegex = / (eq|ne|gt|lt|gte|lte) /;
this.replCompMap = { eq: "==", ne: "!=", gt: ">", lt: "<", gte: ">=", lte: "<=" };

this.replaceComps = function(body){
	var res = null;

	while ((res = this.replCompRegex.exec(body)) != null){
		body = body.replace(res[1], this.replCompMap[res[1]]);
	}
	return body;
}

Downgrade to Node.js v0.8.23! June 10, 2013

Something's going on with 0.10.9 and 0.10.10, where it can't keep my server running. It appears to be related to streams or sockets. But 0.8.23 is working so I will just leave it be until the Node developers can get their acts together :)

Upgrade to Node.js 0.10.10! June 7, 2013

I know the previous post was my upgrade to 0.10.9. But then I started getting an EventEmitter error on Stream end. So I downgraded to 0.8.23 in the meantime, until it was fixed. But then I kept reading, and found out it was fixed in 0.10.10. So I did the git checkout v0.10.10 and make && make install && restart node webserver. After this, it was still getting the error but it wasn't crashing node.js. At this time I saw my error logs flood with file not found (404) errors, and the node.js process was at a consistent 65%. So, I figured it was working and I'll check again when I got home. I'm happy to report, it was just a DDOS attack or something, and the CPU is back down to 2% for Node.js like it should be.

Close call.

Node 0.10.9 June 4, 2013

Upgraded! I'm trying to get all of my computers up to it, so far it's just my mini (netbook) that I use on the couch for dev work, and the server. Left to go = home "real" laptop, and work machine.