Merge pull request #105 from kazimuth/goingnative

Only import the parts of libraries we need.

Author: Jonathan Feinberg

runtime/src/jycessing/LibraryImporter.java

@@ -3,21 +3,30 @@
 import org.python.core.Py;
 import org.python.core.PyObject;
 import org.python.core.PyStringMap;
+import org.python.core.PySystemState;
 import org.python.google.common.base.Joiner;
 import org.python.util.InteractiveConsole;
 
+import processing.core.PApplet;
+
 import java.io.File;
+import java.io.FileFilter;
+import java.io.FileReader;
 import java.io.IOException;
 import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLClassLoader;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Enumeration;
+import java.util.Map;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Properties;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.zip.ZipEntry;
@@ -43,6 +52,9 @@ private static void log(final String msg) {
     }
   }
 
+  private static final String PLATFORM = PApplet.platformNames[PApplet.platform];
+  private static final String BITS = System.getProperty("os.arch").contains("64") ? "64" : "32";
+
   /*
    * Directories where libraries may be found.
    */
@@ -73,68 +85,210 @@ public PyObject __call__(final PyObject[] args, final String[] kws) {
       }
     });
   }
-
+  
+  /**
+   * Locate the library in the library folder "libName", find what
+   * it exports for the current platform, and add exports to the
+   * system classpath, the system native library path, and jython's
+   * sys.path.
+   * 
+   * Then, go through the main jar file of the library and import
+   * all of its publicly exposed classes.
+   * 
+   * @param libName The name of the library to import
+   */
   protected void addLibrary(final String libName) {
     // Don't double-load anything.
     if (loadedLibs.contains(libName)) {
+      log("...never mind, we already did");
       return;
     }
     loadedLibs.add(libName);
+    
 
-    // Find a directory with the given library name in the libSearchPath.
-    File libNameDir = null;
-    for (final File libDir : libSearchPath) {
-      libNameDir = new File(String.format("%s/%s", libDir.getAbsolutePath(), libName));
-      if (libNameDir.exists()) {
-        break;
+    
+    File libDir = null;
+    for (final File searchDir : libSearchPath) {
+      final File potentialDir = new File(searchDir.getAbsoluteFile(), libName);
+      if (potentialDir.exists()) {
+        if (libDir == null) {
+          libDir = potentialDir;
+        } else {
+          System.err.println("Multiple libraries could be " + libName + ";");
+          System.err.println("Picking " + libDir + " over " + potentialDir);
+        }
       }
     }
-    if (libNameDir == null || !libNameDir.exists()) {
-      // Raise the exception in the interpreter, which will give us nice line numbers
-      // when the error is reported in the PDE editor.
+    
+    if (libDir == null) {
       interp.exec("raise Exception('This sketch requires the \"" + libName + "\" library.')");
     }
+    final File contentsDir = new File(libDir, "library");
+    if (!contentsDir.exists()) {
+      interp.exec("raise Exception('The library " + libName + " is malformed and won't import.')");
+    }
+    final File mainJar = new File(contentsDir, libName + ".jar");
+    
+    final List<File> resources = findResources(contentsDir);
+    final PySystemState sys = Py.getSystemState();
+    for (final File resource : resources) {
+      final String name = resource.getName();
+      if (name.endsWith(".jar") || name.endsWith(".zip")) {
+        // Contains stuff we want
+        addJarToClassLoader(resource.getAbsoluteFile());
+        
+        log("Appending " + resource.getAbsolutePath() + " to sys.path.");
+        sys.path.append(Py.newString(resource.getAbsolutePath()));
+        
+        // Are we missing any extensions?
+      } else if (name.endsWith(".so") || name.endsWith(".dll") || name.endsWith(".dylib") || name.endsWith(".jnilib")) {
+        // Add *containing directory* to native search path
+        addDirectoryToNativeSearchPath(resource.getAbsoluteFile().getParentFile());
+      }
+    }
 
-    final File library = new File(libNameDir, "library");
-
-    addToRuntime(library);
-
-    // Find the main library jar file, which can be found at libname/library/libname.jar.
-    final String jarPath = String.format("%s/%s.jar", library.getAbsolutePath(), libName);
     try {
-      importPublicClassesFromJar(jarPath);
-    } catch (final IOException e) {
+      importPublicClassesFromJar(mainJar);
+    } catch (final Exception e) {
       throw new RuntimeException("While trying to add " + libName + " library:", e);
     }
   }
 
   /**
-   * Recursively add the given file to the system classloader, the native lib
-   * search path, and the Jython sys.path.
+   * Find all of the resources a library requires on this platform.
+   * See https://github.com/processing/processing/wiki/Library-Basics.
    * 
-   * <p>The given file should be either a directory or a jar file.
+   * First, finds the library.
+   * Second, tries to parse export.txt, and follow its instructions.
+   * Third, tries to understand folder structure, and export according to that.
    * 
-   * @param file The directory or jar file to make available to sketch runtime.
+   * @param libName The name of the library to add.
+   * @return The list of files we need to import.
    */
-  private void addToRuntime(final File file) {
-    addJarToClassLoader(file);
-    if (file.isDirectory()) {
-      addDirectoryToNativeSearchPath(file);
-    }
-    Py.getSystemState().path.insert(0, Py.newString(file.getAbsolutePath()));
-    if (file.isDirectory()) {
-      for (final File f : file.listFiles()) {
-        if (f.isDirectory() || f.getName().endsWith(".jar")) {
-          addToRuntime(f);
-        }
+  protected List<File> findResources(final File contentsDir) {
+    log("Exploring " + contentsDir + " for resources.");
+    List<File> resources;
+    resources = findResourcesFromExportTxt(contentsDir);
+    if (resources == null) {
+      log("Falling back to directory structure.");
+      resources = findResourcesFromDirectoryStructure(contentsDir);
+    }
+    return resources;
+  }
+  
+  private List<File> findResourcesFromExportTxt(final File contentsDir) {
+    final File exportTxt = new File(contentsDir, "export.txt");
+    if (!exportTxt.exists()) {
+      log("No export.txt in " + contentsDir.getAbsolutePath());
+      return null;
+    }
+    final Map<String, String[]> exportTable;
+    try {
+      exportTable = parseExportTxt(exportTxt);
+    } catch (Exception e) {
+      log("Couldn't parse export.txt: " + e.getMessage());
+      return null;
+    }
+    
+    final String[] resourceNames;
+    
+    // Check from most-specific to least-specific:
+    if (exportTable.containsKey("application." + PLATFORM + BITS)) {
+      log("Found 'application." + PLATFORM + BITS + "' in export.txt");
+      resourceNames = exportTable.get("application." + PLATFORM + BITS);
+    } else if (exportTable.containsKey("application." + PLATFORM)) {
+      log("Found 'application." + PLATFORM + "' in export.txt");
+      resourceNames = exportTable.get("application." + PLATFORM);
+    } else if (exportTable.containsKey("application")) {
+      log("Found 'application' in export.txt");
+      resourceNames = exportTable.get("application");
+    } else {
+      log("No matching platform in " + exportTxt.getAbsolutePath());
+      return null;
+    }
+    final List<File> resources = new ArrayList<>();
+    for (final String resourceName : resourceNames) {
+      final File resource = new File(contentsDir, resourceName);
+      if (resource.exists()) {
+        resources.add(resource);
+      } else {
+        log(resourceName + " is mentioned in " + exportTxt.getAbsolutePath() + "but doesn't actually exist. Moving on.");
+        continue;
+      }
+    }
+    return resources;
+  }
+  
+  private List<File> findResourcesFromDirectoryStructure(final File contentsDir) {
+    final List<String> childNames = Arrays.asList(contentsDir.list());
+    final List<File> resources = new ArrayList<File>();
+    
+    // Find platform-specific stuff
+    File platformDir = null;
+    if (childNames.contains(PLATFORM + BITS)) {
+      final File potentialPlatformDir = new File(contentsDir, PLATFORM + BITS);
+      if (potentialPlatformDir.isDirectory()) {
+        platformDir = potentialPlatformDir;
+      }
+    }
+    if (platformDir == null && childNames.contains(PLATFORM)) {
+      final File potentialPlatformDir = new File(contentsDir, PLATFORM + BITS);
+      if (potentialPlatformDir.isDirectory()) {
+        platformDir = potentialPlatformDir;
+      }
+    }
+    if (platformDir != null) {
+      log("Found platform-specific directory " + platformDir.getAbsolutePath());
+      for (final File resource : platformDir.listFiles()) {
+        resources.add(resource);
+      }
+    }
+    
+    // Find multi-platform stuff; always do this
+    final File[] commonResources = contentsDir.listFiles(new FileFilter() {
+      @Override
+      public boolean accept(File file) {
+        return !file.isDirectory();
+      }
+    });
+    for (final File resource : commonResources) {
+      resources.add(resource);
+    }
+    return resources;
+  }
+  
+  /**
+   * Parse an export.txt file to figure out what we need to load for this platform.
+   * This is all duplicated from processing.app.Library / processing.app.Base,
+   * but we don't have the PDE around at runtime so we can't use them.
+   * 
+   * @param exportTxt The export.txt file; must exist.
+   */
+  private Map<String, String[]> parseExportTxt(final File exportTxt) throws Exception {
+    log("Parsing " + exportTxt.getAbsolutePath());
+    
+    final Properties exportProps = new Properties();
+    try (final FileReader in = new FileReader(exportTxt)) {
+      exportProps.load(in);
+    }
+    
+    final Map<String, String[]> exportTable = new HashMap<>();
+    
+    for (final String platform : exportProps.stringPropertyNames()) {
+      final String exportCSV = exportProps.getProperty(platform);
+      final String[] exports = PApplet.splitTokens(exportCSV, ",");
+      for (int i = 0; i < exports.length; i++) {
+        exports[i] = exports[i].trim();
       }
+      exportTable.put(platform, exports);
     }
+    return exportTable;
   }
 
   /**
    * Use a brittle and egregious hack to forcibly add the given jar file to the
    * system classloader.
-   * @param jar The jar to add to the system clsasloader.
+   * @param jar The jar to add to the system classloader.
    */
   private void addJarToClassLoader(final File jar) {
     try {
@@ -205,7 +359,8 @@ private void addDirectoryToNativeSearchPath(final File dllDir) {
     from com.foo import Banana
     from com.bar import Kiwi
   */
-  private void importPublicClassesFromJar(final String jarPath) throws IOException {
+  private void importPublicClassesFromJar(final File jarPath) throws IOException {
+    log("Importing public classes from " + jarPath.getAbsolutePath());
     try (final ZipFile file = new ZipFile(jarPath)) {
       final Enumeration<? extends ZipEntry> entries = file.entries();
       while (entries.hasMoreElements()) {