Skip to content

Commit 94b16a5

Browse files
author
Federico Fissore
committed
Faster library list downloading by downloading gzipped version
1 parent 365b0bd commit 94b16a5

File tree

6 files changed

+189
-4
lines changed

6 files changed

+189
-4
lines changed

‎app/test/cc/arduino/contributions/GzippedJsonDownloaderTest.java

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package cc.arduino.contributions;
2+
3+
import cc.arduino.contributions.libraries.LibrariesIndex;
4+
import cc.arduino.utils.MultiStepProgress;
5+
import com.fasterxml.jackson.databind.DeserializationFeature;
6+
import com.fasterxml.jackson.databind.ObjectMapper;
7+
import com.fasterxml.jackson.module.mrbean.MrBeanModule;
8+
import org.junit.After;
9+
import org.junit.Before;
10+
import org.junit.Test;
11+
import processing.app.helpers.FileUtils;
12+
13+
import java.io.File;
14+
import java.io.FileInputStream;
15+
import java.io.InputStream;
16+
import java.net.URL;
17+
18+
import static org.junit.Assert.assertTrue;
19+
20+
public class GzippedJsonDownloaderTest {
21+
22+
private File tempFolder;
23+
private File tempFile;
24+
private DownloadableContributionsDownloader downloader;
25+
26+
@Before
27+
public void setUp() throws Exception {
28+
tempFolder = FileUtils.createTempFolder();
29+
tempFile = File.createTempFile("test", ".json");
30+
downloader = new DownloadableContributionsDownloader(tempFolder);
31+
}
32+
33+
@After
34+
public void tearDown() throws Exception {
35+
FileUtils.recursiveDelete(tempFolder);
36+
FileUtils.recursiveDelete(tempFile);
37+
}
38+
39+
@Test
40+
public void testJsonDownload() throws Exception {
41+
new GZippedJsonDownloader(downloader, new URL("http://downloads.arduino.cc/libraries/library_index.json"), new URL("http://downloads.arduino.cc/libraries/library_index.json.gz")).download(tempFile, new MultiStepProgress(1), "");
42+
43+
InputStream indexIn = new FileInputStream(tempFile);
44+
ObjectMapper mapper = new ObjectMapper();
45+
mapper.registerModule(new MrBeanModule());
46+
mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
47+
mapper.configure(DeserializationFeature.EAGER_DESERIALIZER_FETCH, true);
48+
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
49+
LibrariesIndex librariesIndex = mapper.readValue(indexIn, LibrariesIndex.class);
50+
51+
assertTrue(librariesIndex != null);
52+
}
53+
}

‎app/test/cc/arduino/contributions/JsonDownloaderTest.java

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package cc.arduino.contributions;
2+
3+
import cc.arduino.contributions.libraries.LibrariesIndex;
4+
import cc.arduino.utils.MultiStepProgress;
5+
import com.fasterxml.jackson.databind.DeserializationFeature;
6+
import com.fasterxml.jackson.databind.ObjectMapper;
7+
import com.fasterxml.jackson.module.mrbean.MrBeanModule;
8+
import org.junit.After;
9+
import org.junit.Before;
10+
import org.junit.Test;
11+
import processing.app.helpers.FileUtils;
12+
13+
import java.io.File;
14+
import java.io.FileInputStream;
15+
import java.io.InputStream;
16+
import java.net.URL;
17+
18+
import static org.junit.Assert.assertTrue;
19+
20+
public class JsonDownloaderTest {
21+
22+
private File tempFolder;
23+
private File tempFile;
24+
private DownloadableContributionsDownloader downloader;
25+
26+
@Before
27+
public void setUp() throws Exception {
28+
tempFolder = FileUtils.createTempFolder();
29+
tempFile = File.createTempFile("test", ".json");
30+
downloader = new DownloadableContributionsDownloader(tempFolder);
31+
}
32+
33+
@After
34+
public void tearDown() throws Exception {
35+
FileUtils.recursiveDelete(tempFolder);
36+
FileUtils.recursiveDelete(tempFile);
37+
}
38+
39+
@Test
40+
public void testJsonDownload() throws Exception {
41+
new JsonDownloader(downloader, new URL("http://downloads.arduino.cc/libraries/library_index.json")).download(tempFile, new MultiStepProgress(1), "");
42+
43+
InputStream indexIn = new FileInputStream(tempFile);
44+
ObjectMapper mapper = new ObjectMapper();
45+
mapper.registerModule(new MrBeanModule());
46+
mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
47+
mapper.configure(DeserializationFeature.EAGER_DESERIALIZER_FETCH, true);
48+
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
49+
LibrariesIndex librariesIndex = mapper.readValue(indexIn, LibrariesIndex.class);
50+
51+
assertTrue(librariesIndex != null);
52+
}
53+
}

‎arduino-core/src/cc/arduino/contributions/GZippedJsonDownloader.java

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package cc.arduino.contributions;
2+
3+
import cc.arduino.utils.Progress;
4+
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
5+
import org.apache.commons.compress.compressors.gzip.GzipUtils;
6+
import org.apache.commons.compress.utils.IOUtils;
7+
8+
import java.io.*;
9+
import java.net.URL;
10+
11+
public class GZippedJsonDownloader {
12+
13+
private final DownloadableContributionsDownloader downloader;
14+
private final URL url;
15+
private final URL gzippedUrl;
16+
17+
public GZippedJsonDownloader(DownloadableContributionsDownloader downloader, URL url, URL gzippedUrl) {
18+
this.downloader = downloader;
19+
this.url = url;
20+
this.gzippedUrl = gzippedUrl;
21+
}
22+
23+
public void download(File tmpFile, Progress progress, String statusText) throws Exception {
24+
try {
25+
new JsonDownloader(downloader, gzippedUrl).download(tmpFile, progress, statusText);
26+
File gzipTmpFile = new File(tmpFile.getParentFile(), GzipUtils.getCompressedFilename(tmpFile.getName()));
27+
tmpFile.renameTo(gzipTmpFile);
28+
decompress(gzipTmpFile, tmpFile);
29+
} catch (Exception e) {
30+
new JsonDownloader(downloader, url).download(tmpFile, progress, statusText);
31+
}
32+
}
33+
34+
private void decompress(File gzipTmpFile, File tmpFile) throws IOException {
35+
OutputStream os = null;
36+
GzipCompressorInputStream gzipIs = null;
37+
try {
38+
os = new FileOutputStream(tmpFile);
39+
gzipIs = new GzipCompressorInputStream(new FileInputStream(gzipTmpFile));
40+
final byte[] buffer = new byte[4096];
41+
int n = 0;
42+
while (-1 != (n = gzipIs.read(buffer))) {
43+
os.write(buffer, 0, n);
44+
}
45+
} finally {
46+
IOUtils.closeQuietly(os);
47+
IOUtils.closeQuietly(gzipIs);
48+
}
49+
}
50+
}

‎arduino-core/src/cc/arduino/contributions/JsonDownloader.java

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package cc.arduino.contributions;
2+
3+
import cc.arduino.utils.Progress;
4+
5+
import java.io.File;
6+
import java.net.URL;
7+
8+
public class JsonDownloader {
9+
10+
private final DownloadableContributionsDownloader downloader;
11+
private final URL url;
12+
13+
public JsonDownloader(DownloadableContributionsDownloader downloader, URL url) {
14+
this.downloader = downloader;
15+
this.url = url;
16+
}
17+
18+
public void download(File tmpFile, Progress progress, String statusText) throws Exception {
19+
try {
20+
downloader.download(url, tmpFile, progress, statusText);
21+
} catch (InterruptedException e) {
22+
// Download interrupted... just exit
23+
return;
24+
}
25+
}
26+
}

‎arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java

+6-4
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
package cc.arduino.contributions.libraries;
3030

3131
import cc.arduino.contributions.DownloadableContributionsDownloader;
32+
import cc.arduino.contributions.GZippedJsonDownloader;
3233
import cc.arduino.utils.ArchiveExtractor;
3334
import cc.arduino.utils.MultiStepProgress;
3435
import cc.arduino.utils.Progress;
@@ -45,6 +46,7 @@
4546
public class LibraryInstaller {
4647

4748
private static final String LIBRARY_INDEX_URL;
49+
private static final String LIBRARY_INDEX_URL_GZ;
4850

4951
static {
5052
String externalLibraryIndexUrl = System.getProperty("LIBRARY_INDEX_URL");
@@ -53,6 +55,7 @@ public class LibraryInstaller {
5355
} else {
5456
LIBRARY_INDEX_URL = "http://downloads.arduino.cc/libraries/library_index.json";
5557
}
58+
LIBRARY_INDEX_URL_GZ = "http://downloads.arduino.cc/libraries/library_index.json.gz";
5659
}
5760

5861
private final LibrariesIndexer indexer;
@@ -77,8 +80,8 @@ public void updateIndex() throws Exception {
7780
File outputFile = indexer.getIndexFile();
7881
File tmpFile = new File(outputFile.getAbsolutePath() + ".tmp");
7982
try {
80-
downloader.download(url, tmpFile, progress,
81-
_("Downloading libraries index..."));
83+
GZippedJsonDownloader gZippedJsonDownloader = new GZippedJsonDownloader(downloader, new URL(LIBRARY_INDEX_URL), new URL(LIBRARY_INDEX_URL_GZ));
84+
gZippedJsonDownloader.download(tmpFile, progress, _("Downloading libraries index..."));
8285
} catch (InterruptedException e) {
8386
// Download interrupted... just exit
8487
return;
@@ -91,8 +94,7 @@ public void updateIndex() throws Exception {
9194
if (outputFile.exists())
9295
outputFile.delete();
9396
if (!tmpFile.renameTo(outputFile))
94-
throw new Exception(
95-
_("An error occurred while updating libraries index!"));
97+
throw new Exception(_("An error occurred while updating libraries index!"));
9698

9799
// Step 2: Rescan index
98100
rescanLibraryIndex(progress);

‎build/shared/revisions.txt

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ ARDUINO 1.6.5
1010
* Fixed a bug that made the IDE notify users of invalid libraries too many times. Thanks @Chris--A
1111
* Removed JNA. Less native stuff and less chances of incurring into an UnsatisfiedLinkError
1212
* Many new and old issues closed. Thanks to many, and @Chris--A in particular
13+
* Faster libraries list update
1314

1415
[libraries]
1516
* LiquidCrystal fixes. Thanks @newbie15

0 commit comments

Comments
 (0)