All new

alle new

node_modules/lop/lib/StringSource.js

@@ -0,0 +1,53 @@
+var StringSource = module.exports = function(string, description) {
+    var self = {
+        asString: function() {
+            return string;
+        },
+        range: function(startIndex, endIndex) {
+            return new StringSourceRange(string, description, startIndex, endIndex);
+        }
+    };
+    return self;
+};
+
+var StringSourceRange = function(string, description, startIndex, endIndex) {
+    this._string = string;
+    this._description = description;
+    this._startIndex = startIndex;
+    this._endIndex = endIndex;
+};
+
+StringSourceRange.prototype.to = function(otherRange) {
+    // TODO: Assert that tokens are the same across both iterators
+    return new StringSourceRange(this._string, this._description, this._startIndex, otherRange._endIndex);
+};
+
+StringSourceRange.prototype.describe = function() {
+    var position = this._position();
+    var description = this._description ? this._description + "\n" : "";
+    return description + "Line number: " + position.lineNumber + "\nCharacter number: " + position.characterNumber;
+};
+
+StringSourceRange.prototype.lineNumber = function() {
+    return this._position().lineNumber;
+};
+
+StringSourceRange.prototype.characterNumber = function() {
+    return this._position().characterNumber;
+};
+
+StringSourceRange.prototype._position = function() {
+    var self = this;
+    var index = 0;
+    var nextNewLine = function() {
+        return self._string.indexOf("\n", index);
+    };
+
+    var lineNumber = 1;
+    while (nextNewLine() !== -1 && nextNewLine() < this._startIndex) {
+        index = nextNewLine() + 1;
+        lineNumber += 1;
+    }
+    var characterNumber = this._startIndex - index + 1;
+    return {lineNumber: lineNumber, characterNumber: characterNumber};
+};

node_modules/lop/lib/Token.js

@@ -0,0 +1,7 @@
+module.exports = function(name, value, source) {
+    this.name = name;
+    this.value = value;
+    if (source) {
+        this.source = source;
+    }
+};

node_modules/lop/lib/TokenIterator.js

@@ -0,0 +1,28 @@
+var TokenIterator = module.exports = function(tokens, startIndex) {
+    this._tokens = tokens;
+    this._startIndex = startIndex || 0;
+};
+
+TokenIterator.prototype.head = function() {
+    return this._tokens[this._startIndex];
+};
+
+TokenIterator.prototype.tail = function(startIndex) {
+    return new TokenIterator(this._tokens, this._startIndex + 1);
+};
+
+TokenIterator.prototype.toArray = function() {
+    return this._tokens.slice(this._startIndex);
+};
+
+TokenIterator.prototype.end = function() {
+    return this._tokens[this._tokens.length - 1];
+};
+
+// TODO: doesn't need to be a method, can be a separate function,
+// which simplifies implementation of the TokenIterator interface
+TokenIterator.prototype.to = function(end) {
+    var start = this.head().source;
+    var endToken = end.head() || end.end();
+    return start.to(endToken.source);
+};

node_modules/lop/lib/bottom-up.js

@@ -0,0 +1,128 @@
+var rules = require("./rules");
+var results = require("./parsing-results");
+
+exports.parser = function(name, prefixRules, infixRuleBuilders) {
+    var self = {
+        rule: rule,
+        leftAssociative: leftAssociative,
+        rightAssociative: rightAssociative
+    };
+    
+    var infixRules = new InfixRules(infixRuleBuilders.map(createInfixRule));
+    var prefixRule = rules.firstOf(name, prefixRules);
+    
+    function createInfixRule(infixRuleBuilder) {
+        return {
+            name: infixRuleBuilder.name,
+            rule: lazyRule(infixRuleBuilder.ruleBuilder.bind(null, self))
+        };
+    }
+    
+    function rule() {
+        return createRule(infixRules);
+    }
+    
+    function leftAssociative(name) {
+        return createRule(infixRules.untilExclusive(name));
+    }
+    
+    function rightAssociative(name) {
+        return createRule(infixRules.untilInclusive(name));
+    }
+    
+    function createRule(infixRules) {
+        return apply.bind(null, infixRules);
+    }
+    
+    function apply(infixRules, tokens) {
+        var leftResult = prefixRule(tokens);
+        if (leftResult.isSuccess()) {
+            return infixRules.apply(leftResult);
+        } else {
+            return leftResult;
+        }
+    }
+    
+    return self;
+};
+
+function InfixRules(infixRules) {
+    function untilExclusive(name) {
+        return new InfixRules(infixRules.slice(0, ruleNames().indexOf(name)));
+    }
+    
+    function untilInclusive(name) {
+        return new InfixRules(infixRules.slice(0, ruleNames().indexOf(name) + 1));
+    }
+    
+    function ruleNames() {
+        return infixRules.map(function(rule) {
+            return rule.name;
+        });
+    }
+    
+    function apply(leftResult) {
+        var currentResult;
+        var source;
+        while (true) {
+            currentResult = applyToTokens(leftResult.remaining());
+            if (currentResult.isSuccess()) {
+                source = leftResult.source().to(currentResult.source());
+                leftResult = results.success(
+                    currentResult.value()(leftResult.value(), source),
+                    currentResult.remaining(),
+                    source
+                )
+            } else if (currentResult.isFailure()) {
+                return leftResult;
+            } else {
+                return currentResult;
+            }
+        }
+    }
+    
+    function applyToTokens(tokens) {
+        return rules.firstOf("infix", infixRules.map(function(infix) {
+            return infix.rule;
+        }))(tokens);
+    }
+    
+    return {
+        apply: apply,
+        untilExclusive: untilExclusive,
+        untilInclusive: untilInclusive
+    }
+}
+
+exports.infix = function(name, ruleBuilder) {
+    function map(func) {
+        return exports.infix(name, function(parser) {
+            var rule = ruleBuilder(parser);
+            return function(tokens) {
+                var result = rule(tokens);
+                return result.map(function(right) {
+                    return function(left, source) {
+                        return func(left, right, source);
+                    };
+                });
+            };
+        });
+    }
+    
+    return {
+        name: name,
+        ruleBuilder: ruleBuilder,
+        map: map
+    };
+}
+
+// TODO: move into a sensible place and remove duplication
+var lazyRule = function(ruleBuilder) {
+    var rule;
+    return function(input) {
+        if (!rule) {
+            rule = ruleBuilder();
+        }
+        return rule(input);
+    };
+};

node_modules/lop/lib/errors.js

@@ -0,0 +1,22 @@
+exports.error = function(options) {
+    return new Error(options);
+};
+
+var Error = function(options) {
+    this.expected = options.expected;
+    this.actual = options.actual;
+    this._location = options.location;
+};
+
+Error.prototype.describe = function() {
+    var locationDescription = this._location ? this._location.describe() + ":\n" : "";
+    return locationDescription + "Expected " + this.expected + "\nbut got " + this.actual;
+};
+
+Error.prototype.lineNumber = function() {
+    return this._location.lineNumber();
+};
+
+Error.prototype.characterNumber = function() {
+    return this._location.characterNumber();
+};

node_modules/lop/lib/lazy-iterators.js

@@ -0,0 +1,81 @@
+var fromArray = exports.fromArray = function(array) {
+    var index = 0;
+    var hasNext = function() {
+        return index < array.length;
+    };
+    return new LazyIterator({
+        hasNext: hasNext,
+        next: function() {
+            if (!hasNext()) {
+                throw new Error("No more elements");
+            } else {
+                return array[index++];
+            }
+        }
+    });
+};
+
+var LazyIterator = function(iterator) {
+    this._iterator = iterator;
+};
+
+LazyIterator.prototype.map = function(func) {
+    var iterator = this._iterator;
+    return new LazyIterator({
+        hasNext: function() {
+            return iterator.hasNext();
+        },
+        next: function() {
+            return func(iterator.next());
+        }
+    });
+};
+
+LazyIterator.prototype.filter = function(condition) {
+    var iterator = this._iterator;
+    
+    var moved = false;
+    var hasNext = false;
+    var next;
+    var moveIfNecessary = function() {
+        if (moved) {
+            return;
+        }
+        moved = true;
+        hasNext = false;
+        while (iterator.hasNext() && !hasNext) {
+            next = iterator.next();
+            hasNext = condition(next);
+        }
+    };
+    
+    return new LazyIterator({
+        hasNext: function() {
+            moveIfNecessary();
+            return hasNext;
+        },
+        next: function() {
+            moveIfNecessary();
+            var toReturn = next;
+            moved = false;
+            return toReturn;
+        }
+    });
+};
+
+LazyIterator.prototype.first = function() {
+    var iterator = this._iterator;
+    if (this._iterator.hasNext()) {
+        return iterator.next();
+    } else {
+        return null;
+    }
+};
+
+LazyIterator.prototype.toArray = function() {
+    var result = [];
+    while (this._iterator.hasNext()) {
+        result.push(this._iterator.next());
+    }
+    return result;
+};

node_modules/lop/lib/parser.js

@@ -0,0 +1,11 @@
+var TokenIterator = require("./TokenIterator");
+
+exports.Parser = function(options) {
+    var parseTokens = function(parser, tokens) {
+        return parser(new TokenIterator(tokens));
+    };
+    
+    return {
+        parseTokens: parseTokens
+    };
+};

node_modules/lop/lib/parsing-results.js

@@ -0,0 +1,103 @@
+module.exports = {
+    failure: function(errors, remaining) {
+        if (errors.length < 1) {
+            throw new Error("Failure must have errors");
+        }
+        return new Result({
+            status: "failure",
+            remaining: remaining,
+            errors: errors
+        });
+    },
+    error: function(errors, remaining) {
+        if (errors.length < 1) {
+            throw new Error("Failure must have errors");
+        }
+        return new Result({
+            status: "error",
+            remaining: remaining,
+            errors: errors
+        });
+    },
+    success: function(value, remaining, source) {
+        return new Result({
+            status: "success",
+            value: value,
+            source: source,
+            remaining: remaining,
+            errors: []
+        });
+    },
+    cut: function(remaining) {
+        return new Result({
+            status: "cut",
+            remaining: remaining,
+            errors: []
+        });
+    }
+};
+
+var Result = function(options) {
+    this._value = options.value;
+    this._status = options.status;
+    this._hasValue = options.value !== undefined;
+    this._remaining = options.remaining;
+    this._source = options.source;
+    this._errors = options.errors;
+};
+
+Result.prototype.map = function(func) {
+    if (this._hasValue) {
+        return new Result({
+            value: func(this._value, this._source),
+            status: this._status,
+            remaining: this._remaining,
+            source: this._source,
+            errors: this._errors
+        });
+    } else {
+        return this;
+    }
+};
+
+Result.prototype.changeRemaining = function(remaining) {
+    return new Result({
+        value: this._value,
+        status: this._status,
+        remaining: remaining,
+        source: this._source,
+        errors: this._errors
+    });
+};
+
+Result.prototype.isSuccess = function() {
+    return this._status === "success" || this._status === "cut";
+};
+
+Result.prototype.isFailure = function() {
+    return this._status === "failure";
+};
+
+Result.prototype.isError = function() {
+    return this._status === "error";
+};
+
+Result.prototype.isCut = function() {
+    return this._status === "cut";
+};
+
+Result.prototype.value = function() {
+    return this._value;
+};
+
+Result.prototype.remaining = function() {
+    return this._remaining;
+};
+
+Result.prototype.source = function() {
+    return this._source;
+};
+
+Result.prototype.errors = function() {
+    return this._errors;
+};

node_modules/lop/lib/regex-tokeniser.js

@@ -0,0 +1,70 @@
+var Token = require("./Token");
+var StringSource = require("./StringSource");
+
+exports.RegexTokeniser = RegexTokeniser;
+
+function RegexTokeniser(rules) {
+    rules = rules.map(function(rule) {
+        return {
+            name: rule.name,
+            regex: new RegExp(rule.regex.source, "g")
+        };
+    });
+    
+    function tokenise(input, description) {
+        var source = new StringSource(input, description);
+        var index = 0;
+        var tokens = [];
+    
+        while (index < input.length) {
+            var result = readNextToken(input, index, source);
+            index = result.endIndex;
+            tokens.push(result.token);
+        }
+        
+        tokens.push(endToken(input, source));
+        return tokens;
+    }
+
+    function readNextToken(string, startIndex, source) {
+        for (var i = 0; i < rules.length; i++) {
+            var regex = rules[i].regex;
+            regex.lastIndex = startIndex;
+            var result = regex.exec(string);
+            
+            if (result) {
+                var endIndex = startIndex + result[0].length;
+                if (result.index === startIndex && endIndex > startIndex) {
+                    var value = result[1];
+                    var token = new Token(
+                        rules[i].name,
+                        value,
+                        source.range(startIndex, endIndex)
+                    );
+                    return {token: token, endIndex: endIndex};
+                }
+            }
+        }
+        var endIndex = startIndex + 1;
+        var token = new Token(
+            "unrecognisedCharacter",
+            string.substring(startIndex, endIndex),
+            source.range(startIndex, endIndex)
+        );
+        return {token: token, endIndex: endIndex};
+    }
+    
+    function endToken(input, source) {
+        return new Token(
+            "end",
+            null,
+            source.range(input.length, input.length)
+        );
+    }
+    
+    return {
+        tokenise: tokenise
+    }
+}
+
+

node_modules/lop/lib/rules.js

@@ -0,0 +1,313 @@
+var _ = require("underscore");
+var options = require("option");
+var results = require("./parsing-results");
+var errors = require("./errors");
+var lazyIterators = require("./lazy-iterators");
+
+exports.token = function(tokenType, value) {
+    var matchValue = value !== undefined;
+    return function(input) {
+        var token = input.head();
+        if (token && token.name === tokenType && (!matchValue || token.value === value)) {
+            return results.success(token.value, input.tail(), token.source);
+        } else {
+            var expected = describeToken({name: tokenType, value: value});
+            return describeTokenMismatch(input, expected);
+        }
+    };
+};
+
+exports.tokenOfType = function(tokenType) {
+    return exports.token(tokenType);
+};
+
+exports.firstOf = function(name, parsers) {
+    if (!_.isArray(parsers)) {
+        parsers = Array.prototype.slice.call(arguments, 1);
+    }
+    return function(input) {
+        return lazyIterators
+            .fromArray(parsers)
+            .map(function(parser) {
+                return parser(input);
+            })
+            .filter(function(result) {
+                return result.isSuccess() || result.isError();
+            })
+            .first() || describeTokenMismatch(input, name);
+    };
+};
+
+exports.then = function(parser, func) {
+    return function(input) {
+        var result = parser(input);
+        if (!result.map) {
+            console.log(result);
+        }
+        return result.map(func);
+    };
+};
+
+exports.sequence = function() {
+    var parsers = Array.prototype.slice.call(arguments, 0);
+    var rule = function(input) {
+        var result = _.foldl(parsers, function(memo, parser) {
+            var result = memo.result;
+            var hasCut = memo.hasCut;
+            if (!result.isSuccess()) {
+                return {result: result, hasCut: hasCut};
+            }
+            var subResult = parser(result.remaining());
+            if (subResult.isCut()) {
+                return {result: result, hasCut: true};
+            } else if (subResult.isSuccess()) {
+                var values;
+                if (parser.isCaptured) {
+                    values = result.value().withValue(parser, subResult.value());
+                } else {
+                    values = result.value();
+                }
+                var remaining = subResult.remaining();
+                var source = input.to(remaining);
+                return {
+                    result: results.success(values, remaining, source),
+                    hasCut: hasCut
+                };
+            } else if (hasCut) {
+                return {result: results.error(subResult.errors(), subResult.remaining()), hasCut: hasCut};
+            } else {
+                return {result: subResult, hasCut: hasCut};
+            }
+        }, {result: results.success(new SequenceValues(), input), hasCut: false}).result;
+        var source = input.to(result.remaining());
+        return result.map(function(values) {
+            return values.withValue(exports.sequence.source, source);
+        });
+    };
+    rule.head = function() {
+        var firstCapture = _.find(parsers, isCapturedRule);
+        return exports.then(
+            rule,
+            exports.sequence.extract(firstCapture)
+        );
+    };
+    rule.map = function(func) {
+        return exports.then(
+            rule,
+            function(result) {
+                return func.apply(this, result.toArray());
+            }
+        );
+    };
+    
+    function isCapturedRule(subRule) {
+        return subRule.isCaptured;
+    }
+    
+    return rule;
+};
+
+var SequenceValues = function(values, valuesArray) {
+    this._values = values || {};
+    this._valuesArray = valuesArray || [];
+};
+
+SequenceValues.prototype.withValue = function(rule, value) {
+    if (rule.captureName && rule.captureName in this._values) {
+        throw new Error("Cannot add second value for capture \"" + rule.captureName + "\"");
+    } else {
+        var newValues = _.clone(this._values);
+        newValues[rule.captureName] = value;
+        var newValuesArray = this._valuesArray.concat([value]);
+        return new SequenceValues(newValues, newValuesArray);
+    }
+};
+
+SequenceValues.prototype.get = function(rule) {
+    if (rule.captureName in this._values) {
+        return this._values[rule.captureName];
+    } else {
+        throw new Error("No value for capture \"" + rule.captureName + "\"");
+    }
+};
+
+SequenceValues.prototype.toArray = function() {
+    return this._valuesArray;
+};
+
+exports.sequence.capture = function(rule, name) {
+    var captureRule = function() {
+        return rule.apply(this, arguments);
+    };
+    captureRule.captureName = name;
+    captureRule.isCaptured = true;
+    return captureRule;
+};
+
+exports.sequence.extract = function(rule) {
+    return function(result) {
+        return result.get(rule);
+    };
+};
+
+exports.sequence.applyValues = function(func) {
+    // TODO: check captureName doesn't conflict with source or other captures
+    var rules = Array.prototype.slice.call(arguments, 1);
+    return function(result) {
+        var values = rules.map(function(rule) {
+            return result.get(rule);
+        });
+        return func.apply(this, values);
+    };
+};
+
+exports.sequence.source = {
+    captureName: "☃source☃"
+};
+
+exports.sequence.cut = function() {
+    return function(input) {
+        return results.cut(input);
+    };
+};
+
+exports.optional = function(rule) {
+    return function(input) {
+        var result = rule(input);
+        if (result.isSuccess()) {
+            return result.map(options.some);
+        } else if (result.isFailure()) {
+            return results.success(options.none, input);
+        } else {
+            return result;
+        }
+    };
+};
+
+exports.zeroOrMoreWithSeparator = function(rule, separator) {
+    return repeatedWithSeparator(rule, separator, false);
+};
+
+exports.oneOrMoreWithSeparator = function(rule, separator) {
+    return repeatedWithSeparator(rule, separator, true);
+};
+
+var zeroOrMore = exports.zeroOrMore = function(rule) {
+    return function(input) {
+        var values = [];
+        var result;
+        while ((result = rule(input)) && result.isSuccess()) {
+            input = result.remaining();
+            values.push(result.value());
+        }
+        if (result.isError()) {
+            return result;
+        } else {
+            return results.success(values, input);
+        }
+    };
+};
+
+exports.oneOrMore = function(rule) {
+    return exports.oneOrMoreWithSeparator(rule, noOpRule);
+};
+
+function noOpRule(input) {
+    return results.success(null, input);
+}
+
+var repeatedWithSeparator = function(rule, separator, isOneOrMore) {
+    return function(input) {
+        var result = rule(input);
+        if (result.isSuccess()) {
+            var mainRule = exports.sequence.capture(rule, "main");
+            var remainingRule = zeroOrMore(exports.then(
+                exports.sequence(separator, mainRule),
+                exports.sequence.extract(mainRule)
+            ));
+            var remainingResult = remainingRule(result.remaining());
+            return results.success([result.value()].concat(remainingResult.value()), remainingResult.remaining());
+        } else if (isOneOrMore || result.isError()) {
+            return result;
+        } else {
+            return results.success([], input);
+        }
+    };
+};
+
+exports.leftAssociative = function(leftRule, rightRule, func) {
+    var rights;
+    if (func) {
+        rights = [{func: func, rule: rightRule}];
+    } else {
+        rights = rightRule;
+    }
+    rights = rights.map(function(right) {
+        return exports.then(right.rule, function(rightValue) {
+            return function(leftValue, source) {
+                return right.func(leftValue, rightValue, source);
+            };
+        });
+    });
+    var repeatedRule = exports.firstOf.apply(null, ["rules"].concat(rights));
+    
+    return function(input) {
+        var start = input;
+        var leftResult = leftRule(input);
+        if (!leftResult.isSuccess()) {
+            return leftResult;
+        }
+        var repeatedResult = repeatedRule(leftResult.remaining());
+        while (repeatedResult.isSuccess()) {
+            var remaining = repeatedResult.remaining();
+            var source = start.to(repeatedResult.remaining());
+            var right = repeatedResult.value();
+            leftResult = results.success(
+                right(leftResult.value(), source),
+                remaining,
+                source
+            );
+            repeatedResult = repeatedRule(leftResult.remaining());
+        }
+        if (repeatedResult.isError()) {
+            return repeatedResult;
+        }
+        return leftResult;
+    };
+};
+
+exports.leftAssociative.firstOf = function() {
+    return Array.prototype.slice.call(arguments, 0);
+};
+
+exports.nonConsuming = function(rule) {
+    return function(input) {
+        return rule(input).changeRemaining(input);
+    };
+};
+
+var describeToken = function(token) {
+    if (token.value) {
+        return token.name + " \"" + token.value + "\"";
+    } else {
+        return token.name;
+    }
+};
+
+function describeTokenMismatch(input, expected) {
+    var error;
+    var token = input.head();
+    if (token) {
+        error = errors.error({
+            expected: expected,
+            actual: describeToken(token),
+            location: token.source
+        });
+    } else {
+        error = errors.error({
+            expected: expected,
+            actual: "end of tokens"
+        });
+    }
+    return results.failure([error], input);
+}

node_modules/lop/lib/testing.js

@@ -0,0 +1,69 @@
+var _ = require("underscore");
+var duck = require("duck");
+
+var Token = require("./Token");
+
+var assertIsSuccess = exports.assertIsSuccess = function(test, result, options) {
+    test.ok(result.isSuccess(), result.errors().map(function(error) {
+        return error.describe();
+    }).join("\n\n"));
+    options = options || {};
+    if (!options.remaining) {
+        options.remaining = [new Token("end", null)];
+    }
+    assertResultExtras(test, result, options);
+};
+
+var assertIsSuccessWithValue = exports.assertIsSuccessWithValue = function(test, result, value, source) {
+    assertIsSuccess(test, result, {
+        value: value,
+        source: source
+    });
+};
+
+var assertIsFailure = exports.assertIsFailure = function(test, result, options) {
+    test.deepEqual(result.isSuccess(), false);
+    assertResultExtras(test, result, options);
+};
+
+var assertRemaining = exports.assertRemaining = function(test, result, expectedRemaining) {
+    var actualRemaining = result.remaining().toArray();
+    assertThat(test, actualRemaining, duck.isArray(expectedRemaining.map(duck.hasProperties)));
+};
+
+var assertErrors = exports.assertErrors = function(test, result, expectedErrors) {
+    test.deepEqual(result.errors(), expectedErrors);
+};
+
+var assertIsFailureWithRemaining = exports.assertIsFailureWithRemaining = function(test, result, expectedRemaining) {
+    assertIsFailure(test, result);
+    assertRemaining(test, result, expectedRemaining);
+};
+
+var assertIsError = exports.assertIsError = function(test, result, options) {
+    test.ok(result.isError());
+    assertResultExtras(test, result, options);
+};
+
+var assertResultExtras = function(test, result, options) {
+    options = options || {};
+    if (options.remaining) {
+        assertRemaining(test, result, options.remaining);
+    }
+    if (options.errors) {
+        assertErrors(test, result, options.errors);
+    }
+    if (options.source) {
+        test.deepEqual(result.source(), options.source);
+    }
+    if (options.value) {
+        assertThat(test, result.value(), duck.is(options.value));
+    }
+}
+
+var assertThat = function(test, value, matcher) {
+    var message = "Expected " + matcher.describeSelf() +
+        "\nbut " + matcher.describeMismatch(value);
+    test.ok(matcher.matches(value), message);
+};
+