Add emoji data

We can do this now that the emoji data is versioned in the same way as the Unicode Standard itself.

Author: mathiasbynens

data/11.0.0-emoji-sequences.txt

@@ -273,7 +273,13 @@ var resources = [
 		'bidi-mirroring': 'https://unicode.org/Public/11.0.0/ucd/BidiMirroring-11.0.0d3.txt',
 		'bidi-brackets': 'https://unicode.org/Public/11.0.0/ucd/BidiBrackets-11.0.0d1.txt',
 		'line-break': 'https://unicode.org/Public/11.0.0/ucd/LineBreak-11.0.0d10.txt',
-		'word-break': 'https://unicode.org/Public/11.0.0/ucd/auxiliary/WordBreakProperty-11.0.0d20.txt'
+		'word-break': 'https://unicode.org/Public/11.0.0/ucd/auxiliary/WordBreakProperty-11.0.0d20.txt',
+		// Emoji, Emoji_Presentation, Emoji_Modifier, Emoji_Modifier_Base, Emoji_Component, Extended_Pictographic
+		'emoji': 'https://unicode.org/Public/emoji/11.0/emoji-data.txt',
+		// Emoji_Keycap_Sequence, Emoji_Flag_Sequence, Emoji_Modifier_Sequence,
+		'emoji-sequences': 'https://unicode.org/Public/emoji/11.0/emoji-sequences.txt',
+		// Emoji_ZWJ_Sequence
+		'emoji-zwj-sequences': 'https://unicode.org/Public/emoji/11.0/emoji-zwj-sequences.txt',
 	},
 ];
 

data/11.0.0-emoji-zwj-sequences.txt

@@ -11,6 +11,8 @@ parsers.parseCompositionExclusions = require('./scripts/parse-composition-exclus
 parsers.parseLineBreak = require('./scripts/parse-line-break.js');
 parsers.parseScriptExtensions = require('./scripts/parse-script-extensions.js');
 parsers.parseWordBreak = require('./scripts/parse-word-break.js');
+parsers.parseEmoji = require('./scripts/parse-emoji.js');
+parsers.parseEmojiSequences = require('./scripts/parse-emoji-sequences.js');
 parsers.parseNames = require('./scripts/parse-names.js');
 const extend = utils.extend;
 const cp = require('cp');
@@ -119,6 +121,18 @@ const generateData = function(version) {
 		'map': parsers.parseWordBreak(version),
 		'type': 'Word_Break'
 	}));
+	console.log('Parsing Unicode v%s binary emoji properties…', version);
+	extend(dirMap, utils.writeFiles({
+		'version': version,
+		'map': parsers.parseEmoji(version),
+		'type': 'Binary_Property'
+	}));
+	console.log('Parsing Unicode v%s emoji sequence properties…', version);
+	extend(dirMap, utils.writeFiles({
+		'version': version,
+		'map': parsers.parseEmojiSequences(version),
+		'type': 'Sequence_Property'
+	}));
 	console.log('Parsing Unicode v%s `Names`…', version);
 	extend(dirMap, utils.writeFiles({
 		'version': version,

data/11.0.0-emoji.txt

@@ -40,7 +40,10 @@ const TYPES = [
 	'bidi-mirroring',
 	'bidi-brackets',
 	'line-break',
-	'word-break'
+	'word-break',
+	'emoji',
+	'emoji-sequences',
+	'emoji-zwj-sequences'
 ];
 
 for (const resource of resources) {

data/resources.js

@@ -0,0 +1,35 @@
+'use strict';
+
+const utils = require('./utils.js');
+
+const parseEmojiSequences = function(version) {
+	const source = utils.readDataFile(version, 'emoji-sequences');
+	if (!source) {
+		return;
+	}
+	const propertyMap = new Map();
+	const lines = source.split('\n');
+	lines.forEach(function(line) {
+		if (!line || /^#/.test(line)) {
+			return;
+		}
+		const data = line.trim().split('; ');
+		const codePoints = data[0].trim().split(' ').map((hex) => {
+			return parseInt(hex, 16);
+		});
+		const sequence = String.fromCodePoint(...codePoints);
+		const property = data[1].split('#')[0].trim();
+		if (propertyMap.has(property)) {
+			propertyMap.get(property).add(sequence);
+		} else {
+			propertyMap.set(property, new Set([sequence]));
+		}
+	});
+	const plainObject = {};
+	for (const [property, codePoints] of propertyMap) {
+		plainObject[property] = [...codePoints].sort((a, b) => a - b);
+	}
+	return plainObject;
+};
+
+module.exports = parseEmojiSequences;

index.js

@@ -0,0 +1,47 @@
+'use strict';
+
+const utils = require('./utils.js');
+
+const parseEmoji = function(version) {
+	const source = utils.readDataFile(version, 'emoji');
+	if (!source) {
+		return;
+	}
+	const propertyMap = new Map();
+	const lines = source.split('\n');
+	lines.forEach(function(line) {
+		if (!line || /^#/.test(line)) {
+			return;
+		}
+		const data = line.trim().split(' ; ');
+		const charRange = data[0].replace('..', '-').trim();
+		const rangeParts = charRange.split('-');
+		const property = data[1].split('#')[0].trim();
+		if (rangeParts.length == 2) {
+			utils.range(
+				parseInt(rangeParts[0], 16),
+				parseInt(rangeParts[1], 16)
+			).forEach(function(codePoint) {
+				if (propertyMap.has(property)) {
+					propertyMap.get(property).add(codePoint);
+				} else {
+					propertyMap.set(property, new Set([codePoint]));
+				}
+			});
+		} else {
+			const codePoint = parseInt(rangeParts, 16);
+			if (propertyMap.has(property)) {
+				propertyMap.get(property).add(codePoint);
+			} else {
+				propertyMap.set(property, new Set([codePoint]));
+			}
+		}
+	});
+	const plainObject = {};
+	for (const [property, codePoints] of propertyMap) {
+		plainObject[property] = [...codePoints].sort((a, b) => a - b);
+	}
+	return plainObject;
+};
+
+module.exports = parseEmoji;

scripts/download.js

@@ -83,8 +83,20 @@ const writeFiles = function(options) {
 			return;
 		}
 		append(dirMap, type, item);
-		// Create the target directory if it doesn’t exist yet
+		// Create the target directory if it doesn’t exist yet.
 		mkdirp.sync(dir);
+
+		// Sequence properties are special.
+		if (type == 'Sequence_Property') {
+			const sequences = codePoints;
+			const output = `module.exports=${ gzipInline(map[item]) }`;
+			fs.writeFileSync(
+				path.resolve(dir, 'index.js'),
+				output
+			);
+			return;
+		}
+
 		// Save the data to a file
 		fs.writeFileSync(
 			path.resolve(dir, 'code-points.js'),

scripts/parse-emoji-sequences.js

@@ -71,6 +71,10 @@ require('unicode-<%= version %>/<%= type %>/<%= dir %>/code-points.js'); // look
 require('unicode-<%= version %>/<%= type %>/<%= dir %>/code-points.js').get(codePoint);
 require('unicode-<%= version %>/<%= type %>/<%= dir %>/symbols.js'); // lookup map from symbol to symbol(s)
 require('unicode-<%= version %>/<%= type %>/<%= dir %>/symbols.js').get(symbol);
+<%
+		} else if ('Sequence_Property' == type) {
+%>
+require('unicode-<%= version %>/<%= type %>/<%= dir %>/index.js'); // array containing a string for each sequence
 <%
 		} else {
 %>