diff --git a/org.eclipse.lsp4j/src/main/java/org/eclipse/lsp4j/util/SemanticHighlightingTokens.java b/org.eclipse.lsp4j/src/main/java/org/eclipse/lsp4j/util/SemanticHighlightingTokens.java new file mode 100644 index 000000000..b0b2c6793 --- /dev/null +++ b/org.eclipse.lsp4j/src/main/java/org/eclipse/lsp4j/util/SemanticHighlightingTokens.java @@ -0,0 +1,237 @@ +/******************************************************************************* + * Copyright (c) 2018 TypeFox GmbH (http://www.typefox.io) and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + *******************************************************************************/ +package org.eclipse.lsp4j.util; + +import static java.util.Collections.emptyList; + +import java.nio.ByteBuffer; +import java.util.Base64; +import java.util.List; + +import org.eclipse.xtext.xbase.lib.IterableExtensions; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ComparisonChain; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.Iterables; + +/** + * Utility class for encoding and decoding semantic highlighting tokens into a + * compact, {@code base64} representation. + */ +public final class SemanticHighlightingTokens { + + private static int LENGTH_SHIFT = 0x0000010; + private static int SCOPES_MASK = 0x0000FFFF; + + /** + * Encodes the iterable of tokens into a compact, {@code base64} string. Returns + * with an empty string if the {@code tokens} argument is {@code null} or empty. + */ + public static String encode(Iterable tokens) { + if (IterableExtensions.isNullOrEmpty(tokens)) { + return ""; + } + // 2 stands for: number of elements for the output; the "character" and the + // "length and scope". + // 4 stands for: 4 byte for a primitive integer. + ByteBuffer buffer = ByteBuffer.allocate(Iterables.size(tokens) * 2 * 4); + for (Token token : tokens) { + int character = token.character; + int length = token.length; + int scope = token.scope; + + int lengthAndScope = length; + lengthAndScope = lengthAndScope << LENGTH_SHIFT; + lengthAndScope |= scope; + + buffer.putInt(character); + buffer.putInt(lengthAndScope); + } + return Base64.getEncoder().encodeToString(buffer.array()); + } + + /** + * Decodes the tokens string and returns with a list of semantic highlighting + * tokens. Returns with an empty list if the argument is {@code null} or empty. + */ + public static List decode(String tokens) { + if (tokens == null || tokens.length() == 0) { + return emptyList(); + } + ByteBuffer buffer = ByteBuffer.wrap(Base64.getDecoder().decode(tokens)); + Builder builder = ImmutableList.builder(); + while (buffer.hasRemaining()) { + int character = buffer.getInt(); + + int lengthAndScope = buffer.getInt(); + int length = lengthAndScope >>> LENGTH_SHIFT; + int scope = lengthAndScope & SCOPES_MASK; + + builder.add(new Token(character, length, scope)); + } + return builder.build(); + } + + /** + * The bare minimum representation of a semantic highlighting token. + * + *

+ * Semantic highlighting tokens are not part of the protocol, as they're + * sent through the wire as an encoded string. The purpose of this data type is + * to help working with them. + */ + public static class Token implements Comparable { + + private static int[] EMPTY_ARRAY = new int[0]; + private static int MAX_UINT_16 = 65_535; // 2^16 - 1 + + /** + * Converts an iterable of tokens into an array of primitive integers. Returns + * with an empty array if the argument is {@code null} or empty. + * + *

+ *

+ */ + public static int[] toIntArray(Iterable tokens) { + if (IterableExtensions.isNullOrEmpty(tokens)) { + return EMPTY_ARRAY; + } + int[] array = new int[Iterables.size(tokens) * 3]; + int i = 0; + for (Token token : tokens) { + array[i++] = token.character; + array[i++] = token.length; + array[i++] = token.scope; + } + return array; + } + + /** + * Counter-part of the {@link Token#toIntArray}. Converts an array of primitive + * integers into a list of tokens. If the input is {@code null}, returns with an + * empty list. Throws an exception if the length of the arguments is not a + * modulo of {@code 3}. + * + * The elements of the input will be used to create new {@link Token} instances, + * hence they must comply the requirements described + * {@link #Token(int, int, int) here}. + */ + public static List fromIntArray(int... input) { + if (input == null) { + return emptyList(); + } + int inputLength = input.length; + Preconditions.checkArgument(inputLength % 3 == 0, + "Invalid input. 'input.length % 3 != 0' Input length was: " + inputLength + "."); + Builder builder = ImmutableList.builder(); + for (int i = 0; i < inputLength; i = i + 3) { + builder.add(new Token( + input[i], // character + input[i + 1], // length + input[i + 2] // scope + )); + } + return builder.build(); + } + + /** + * The zero-based character offset of the token in the line. + */ + public final int character; + + /** + * The length of the token. + */ + public final int length; + + /** + * The internal index of the + * TextMate + * scope. + */ + public final int scope; + + /** + * Creates a new highlighting token. + * + * @param character + * the character offset of the token. Must not be a negative integer. + * @param length + * the length of the token. Must be an integer between {@code 0} and + * {@code 2}16{@code -1} (inclusive). + * @param scope + * the scope. Must be an integer between {@code 0} and + * {@code 2}16{@code -1} (inclusive). + */ + public Token(int character, int length, int scope) { + Preconditions.checkArgument(character >= 0, "Expected 'character' >= 0. Was: " + character + "."); + this.character = character; + this.length = assertUint16(length, "length"); + this.scope = assertUint16(scope, "scope"); + } + + /** + * Asserts the argument. It must be in a range of {@code 0} and + * {@code 2}16{@code -1} (inclusive). Returns with the argument if + * valid. Otherwise throws an {@link IllegalArgumentException}. + */ + private static int assertUint16(int i, String prefix) { + Preconditions.checkArgument(i >= 0, "Expected: '" + prefix + "' >= 0. '" + prefix + "' was: " + i + "."); + Preconditions.checkArgument(i <= MAX_UINT_16, + "Expected: '" + prefix + "' <= " + MAX_UINT_16 + ". '" + prefix + "' was: " + i + "."); + return i; + } + + @Override + public int compareTo(Token o) { + return ComparisonChain.start().compare(character, o.character).compare(length, o.length) + .compare(scope, o.scope).result(); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + character; + result = prime * result + length; + result = prime * result + scope; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Token other = (Token) obj; + if (character != other.character) + return false; + if (length != other.length) + return false; + if (scope != other.scope) + return false; + return true; + } + + } + + private SemanticHighlightingTokens() { + } + +} diff --git a/org.eclipse.lsp4j/src/test/java/org/eclipse/lsp4j/test/util/SemanticHighlightingTokensTest.java b/org.eclipse.lsp4j/src/test/java/org/eclipse/lsp4j/test/util/SemanticHighlightingTokensTest.java new file mode 100644 index 000000000..ccf1fb3f8 --- /dev/null +++ b/org.eclipse.lsp4j/src/test/java/org/eclipse/lsp4j/test/util/SemanticHighlightingTokensTest.java @@ -0,0 +1,141 @@ +/******************************************************************************* + * Copyright (c) 2018 TypeFox GmbH (http://www.typefox.io) and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + *******************************************************************************/ +package org.eclipse.lsp4j.test.util; + +import static com.google.common.collect.Lists.newArrayList; +import static java.util.Collections.emptyList; +import static org.eclipse.lsp4j.util.SemanticHighlightingTokens.decode; +import static org.eclipse.lsp4j.util.SemanticHighlightingTokens.encode; +import static org.eclipse.lsp4j.util.SemanticHighlightingTokens.Token.fromIntArray; +import static org.eclipse.lsp4j.util.SemanticHighlightingTokens.Token.toIntArray; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.util.List; + +import org.eclipse.lsp4j.util.SemanticHighlightingTokens.Token; +import org.junit.Test; + +public class SemanticHighlightingTokensTest { + + private static int[] EMPTY_ARRAY = new int[0]; + + @Test(expected = IllegalArgumentException.class) + public void newToken_invalidCharacter() { + new Token(-1, 0, 0); + } + + @Test(expected = IllegalArgumentException.class) + public void newToken_invalidLengthSmaller() { + new Token(0, -1, 0); + } + + @Test(expected = IllegalArgumentException.class) + public void newToken_invalidLengthGreater() { + new Token(0, (int) Math.pow(2, 16), 0); + } + + @Test(expected = IllegalArgumentException.class) + public void newToken_invalidScopeSmaller() { + new Token(0, 0, -1); + } + + @Test(expected = IllegalArgumentException.class) + public void newToken_invalidScopeGreater() { + new Token(0, 0, (int) Math.pow(2, 16)); + } + + @Test + public void toIntArray_null() { + assertArrayEquals(EMPTY_ARRAY, toIntArray(null)); + } + + @Test + public void toIntArray_empty() { + assertArrayEquals(EMPTY_ARRAY, toIntArray(emptyList())); + } + + @Test(expected = NullPointerException.class) + public void toIntArray_invalid() { + toIntArray(newArrayList((Token) null)); + } + + @Test + public void toIntArray_() { + assertArrayEquals(new int[] { 2, 5, 0, 12, 15, 1, 7, 1000, 1 }, + toIntArray(newArrayList(new Token(2, 5, 0), new Token(12, 15, 1), new Token(7, 1000, 1)))); + } + + @Test + public void fromIntArray_null() { + assertEquals(emptyList(), fromIntArray(null)); + } + + @Test + public void fromIntArray_empty() { + assertEquals(emptyList(), fromIntArray(EMPTY_ARRAY)); + } + + @Test(expected = IllegalArgumentException.class) + public void fromIntArray_invalid() { + assertEquals(emptyList(), fromIntArray(new int[] { 1, 2 })); + } + + @Test + public void fromIntArray_() { + assertEquals(newArrayList(new Token(2, 5, 0), new Token(12, 15, 1), new Token(7, 1000, 1)), + fromIntArray(new int[] { 2, 5, 0, 12, 15, 1, 7, 1000, 1 })); + } + + @Test + public void encode_null() { + assertEquals("", encode(null)); + } + + @Test + public void encode_empty() { + assertEquals("", encode(emptyList())); + } + + @Test(expected = NullPointerException.class) + public void encode_invalid() { + encode(newArrayList((Token) null)); + } + + @Test + public void encode_() { + assertEquals("AAAAAgAFAAAAAAAMAA8AAQAAAAcD6AAB", + encode(newArrayList(new Token(2, 5, 0), new Token(12, 15, 1), new Token(7, 1000, 1)))); + } + + @Test + public void decode_null() { + assertEquals(emptyList(), decode(null)); + } + + @Test + public void decode_empty() { + assertEquals(emptyList(), decode("")); + } + + @Test + public void decode_() { + assertEquals(newArrayList(new Token(2, 5, 0), new Token(12, 15, 1), new Token(7, 1000, 1)), + decode("AAAAAgAFAAAAAAAMAA8AAQAAAAcD6AAB")); + } + + @Test + public void symmetric() { + List expected = newArrayList(); + for (int i = 0; i < 65536; i++) { + expected.add(new Token(i, i, i)); + } + assertEquals(expected, decode(encode(expected))); + } + +}