Skip to content

Advanced message parsing #34

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Advanced message parsing #34

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cab9299
feat(Parser): allow left-padded values
leon0399 Feb 27, 2024
aff3e74
feat(Parser): allow equals (`=`) sign for label-value separator
leon0399 Feb 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 43 additions & 18 deletions src/msgAggregatorWorker.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,24 +81,49 @@ describe("Parsing data", () => {
);
});

test("labeled", () => {
const messages = [
`0${trailingFieldDelimiter}${recordDelimiter}`,
`label_1:1${fieldDelimiter}label_2:2${trailingFieldDelimiter}${recordDelimiter}`,
`label_1:3${fieldDelimiter}label_2:4${trailingFieldDelimiter}${recordDelimiter}`,
];

const assertion = {
datasetNames: ["label_1", "label_2"],
parsedLines: [
{ label_1: 1, label_2: 2 },
{ label_1: 3, label_2: 4 },
],
};

expect(messageAggregator.parseSerialMessages(messages)).toEqual(
assertion
);
describe.each([
["colon", ":"],
["equals", "="],
])("%s label delimiter", (_, labelDelimiter) => {
test("labeled", () => {
const messages = [
`0${trailingFieldDelimiter}${recordDelimiter}`,
`label_1${labelDelimiter}1${fieldDelimiter}label_2${labelDelimiter}2${trailingFieldDelimiter}${recordDelimiter}`,
`label_1${labelDelimiter}3${fieldDelimiter}label_2${labelDelimiter}4${trailingFieldDelimiter}${recordDelimiter}`,
];

const assertion = {
datasetNames: ["label_1", "label_2"],
parsedLines: [
{ label_1: 1, label_2: 2 },
{ label_1: 3, label_2: 4 },
],
};

expect(messageAggregator.parseSerialMessages(messages)).toEqual(
assertion
);
});

test("labeled padded", () => {
const messages = [
`0${trailingFieldDelimiter}${recordDelimiter}`,
`label_1${labelDelimiter} 1${fieldDelimiter}label_2${labelDelimiter} 20${trailingFieldDelimiter}${recordDelimiter}`,
`label_1${labelDelimiter} 300${fieldDelimiter}label_2${labelDelimiter}4000${trailingFieldDelimiter}${recordDelimiter}`,
];

const assertion = {
datasetNames: ["label_1", "label_2"],
parsedLines: [
{ label_1: 1, label_2: 20 },
{ label_1: 300, label_2: 4000 },
],
};

expect(messageAggregator.parseSerialMessages(messages)).toEqual(
assertion
);
});
});

test("buffering", () => {
Expand Down
42 changes: 27 additions & 15 deletions src/msgAggregatorWorker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ ctx.addEventListener("message", (event) => {

let buffer = "";
let discardFirstLine = true;
const separator = "\r?\n";
const lineSeparator = "\r?\n";
const delimiter = "[, \t]+"; // Serial Plotter protocol supports Comma, Space & Tab characters as delimiters
var separatorRegex = new RegExp(`(${separator})`, "g");
var lineSeparatorRegex = new RegExp(`(${lineSeparator})`, "g");
var delimiterRegex = new RegExp(delimiter, "g");

export const parseSerialMessages = (
Expand All @@ -33,8 +33,8 @@ export const parseSerialMessages = (
// so we need to discard it and start aggregating from the first encountered separator
let joinMessages = messages.join("");
if (discardFirstLine) {
separatorRegex.lastIndex = 0; // Reset lastIndex to ensure match happens from beginning of string
const separatorMatch = separatorRegex.exec(joinMessages);
lineSeparatorRegex.lastIndex = 0; // Reset lastIndex to ensure match happens from beginning of string
const separatorMatch = lineSeparatorRegex.exec(joinMessages);
if (separatorMatch && separatorMatch.index > -1) {
joinMessages = joinMessages.substring(
separatorMatch.index + separatorMatch[0].length
Expand All @@ -50,14 +50,16 @@ export const parseSerialMessages = (

//add any leftover from the buffer to the first line
const messagesAndBuffer = ((buffer || "") + joinMessages)
.split(separatorRegex)
.split(lineSeparatorRegex)
.filter((message) => message.length > 0);

// remove the previous buffer
buffer = "";
separatorRegex.lastIndex = 0;
lineSeparatorRegex.lastIndex = 0;
// check if the last message contains the delimiter, if not, it's an incomplete string that needs to be added to the buffer
if (!separatorRegex.test(messagesAndBuffer[messagesAndBuffer.length - 1])) {
if (
!lineSeparatorRegex.test(messagesAndBuffer[messagesAndBuffer.length - 1])
) {
buffer = messagesAndBuffer[messagesAndBuffer.length - 1];
messagesAndBuffer.splice(-1);
}
Expand All @@ -66,10 +68,17 @@ export const parseSerialMessages = (
const parsedLines: { [key: string]: number }[] = [];

// for each line, explode variables
separatorRegex.lastIndex = 0;
lineSeparatorRegex.lastIndex = 0;
messagesAndBuffer
.filter((message) => !separatorRegex.test(message))
.filter((message) => !lineSeparatorRegex.test(message))
.forEach((message) => {
// replace all delimiters with a single space for uniform parsing
message = message.replace(delimiterRegex, " ");
// replace multiple spaces with a single space
message = message.replace(/\s+/g, " ");
// replace all equal signs with a colon
message = message.replace(/=/g, ":");

const parsedLine: { [key: string]: number } = {};

// Part Separator symbols i.e. Space, Tab & Comma are fully supported
Expand All @@ -80,12 +89,15 @@ export const parseSerialMessages = (
// if we find a colon, we assume the latter is being used
let tokens: string[] = [];
if (message.indexOf(":") > 0) {
message.split(delimiterRegex).forEach((keyValue: string) => {
let [key, value] = keyValue.split(":");
key = key && key.trim();
value = value && value.trim();
if (key && key.length > 0 && value && value.length > 0) {
tokens.push(...[key, value]);
// Splitting by the separator and handling possible spaces
const keyValuePairs = message.split(":").map((kv) => kv.trim());
let reformedLine = keyValuePairs.join(":").split(delimiterRegex);

reformedLine.forEach((kv) => {
const [key, value] = kv.split(":");
if (key && value) {
tokens.push(key.trim());
tokens.push(value.trim());
}
});
} else {
Expand Down