[jruby~main:6857a4e5] Add codecache support for precompiled scripts dumping to disk and loading

[nicksieger@kenai.com]

Project:    jruby
Repository: main
Revision:   6857a4e5c345a1b90d1832ba70d03d04e21aea04
Author:     nicksieger
Date:       2010-01-21 18:20:29 UTC
Link:       

Log Message:
------------
Add codecache support for precompiled scripts dumping to disk and loading 
from classloaders.


Revisions:
----------
6857a4e5c345a1b90d1832ba70d03d04e21aea04


Modified Paths:
---------------
src/org/jruby/Ruby.java
src/org/jruby/compiler/JITCompiler.java


Diffs:
------
diff --git a/src/org/jruby/Ruby.java b/src/org/jruby/Ruby.java
index c2772e9..8c6d8c0 100644
--- a/src/org/jruby/Ruby.java
+++ b/src/org/jruby/Ruby.java
@@ -127,6 +127,9 @@ import org.jruby.util.io.ChannelDescriptor;
 import com.kenai.constantine.Constant;
 import com.kenai.constantine.ConstantSet;
 import com.kenai.constantine.platform.Errno;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.File;
 import java.util.EnumSet;
 import java.util.concurrent.atomic.AtomicLong;
 import org.jruby.ast.RootNode;
@@ -556,7 +559,7 @@ public final class Ruby {
         boolean compile = 
getInstanceConfig().getCompileMode().shouldPrecompileCLI();
         boolean forceCompile = 
getInstanceConfig().getCompileMode().shouldPrecompileAll();
         if (compile) {
-            script = tryCompile(scriptNode, new 
JRubyClassLoader(getJRubyClassLoader()), config.isShowBytecode());
+            script = tryCompile(scriptNode, null, new 
JRubyClassLoader(getJRubyClassLoader()), config.isShowBytecode());
             if (forceCompile && script == null) {
                 return getNil();
             }
@@ -575,14 +578,14 @@ public final class Ruby {
     }
     
     public Script tryCompile(Node node) {
-        return tryCompile(node, new JRubyClassLoader(getJRubyClassLoader()));
+        return tryCompile(node, null, new 
JRubyClassLoader(getJRubyClassLoader()));
     }
     
-    private Script tryCompile(Node node, JRubyClassLoader classLoader) {
-        return tryCompile(node, classLoader, false);
+    private Script tryCompile(Node node, String cachedClassName, 
JRubyClassLoader classLoader) {
+        return tryCompile(node, cachedClassName, classLoader, false);
     }
 
-    private Script tryCompile(Node node, JRubyClassLoader classLoader, 
boolean dump) {
+    private Script tryCompile(Node node, String cachedClassName, 
JRubyClassLoader classLoader, boolean dump) {
         Script script = null;
         try {
             String filename = node.getPosition().getFile();
@@ -591,7 +594,13 @@ public final class Ruby {
             ASTInspector inspector = new ASTInspector();
             inspector.inspect(node);
 
-            StandardASMCompiler asmCompiler = new 
StandardASMCompiler(classname, filename);
+            StandardASMCompiler asmCompiler = null;
+            if (RubyInstanceConfig.JIT_CODE_CACHE != null && cachedClassName 
!= null) {
+                System.out.println(cachedClassName);
+                asmCompiler = new 
StandardASMCompiler(cachedClassName.replace('.', '/'), filename);
+            } else {
+                asmCompiler = new StandardASMCompiler(classname, filename);
+            }
             ASTCompiler compiler = config.newCompiler();
             if (dump) {
                 compiler.compileRoot(node, asmCompiler, inspector, false, 
false);
@@ -599,6 +608,13 @@ public final class Ruby {
             } else {
                 compiler.compileRoot(node, asmCompiler, inspector, true, 
false);
             }
+
+            if (RubyInstanceConfig.JIT_CODE_CACHE != null && cachedClassName 
!= null) {
+                // save script off to disk
+                String pathName = cachedClassName.replace('.', '/');
+                JITCompiler.saveToCodeCache(this, 
asmCompiler.getClassByteArray(), "ruby/jit", new 
File(RubyInstanceConfig.JIT_CODE_CACHE, pathName + ".class"));
+            }
+
             script = 
(Script)asmCompiler.loadClass(classLoader).newInstance();
 
             if (config.isJitLogging()) {
@@ -2504,16 +2520,62 @@ public final class Ruby {
         IRubyObject self = wrap ? TopSelfFactory.createTopSelf(this) : 
getTopSelf();
         ThreadContext context = getCurrentContext();
         String file = context.getFile();
+        InputStream readStream = in;
         
         try {
             secure(4); /* should alter global state */
 
             context.setFile(filename);
             context.preNodeEval(objectClass, self, filename);
+
+            Script script = null;
+            String className = null;
+
+            try {
+                // read full contents of file, hash it, and try to load that 
class first
+                ByteArrayOutputStream baos = new ByteArrayOutputStream();
+                byte[] buffer = new byte[1024];
+                int num;
+                while ((num = in.read(buffer)) > -1) {
+                    baos.write(buffer, 0, num);
+                }
+                buffer = baos.toByteArray();
+                String hash = JITCompiler.getHashForBytes(buffer);
+                className = "ruby.jit.FILE_" + hash;
+
+                // FIXME: duplicated from ClassCache
+                Class contents;
+                try {
+                    contents = jrubyClassLoader.loadClass(className);
+                    if (JITCompiler.DEBUG) {
+                        System.err.println("found jitted code in 
classloader: " + className);
+                    }
+                    script = (Script)contents.newInstance();
+                    readStream = new ByteArrayInputStream(buffer);
+                } catch (ClassNotFoundException cnfe) {
+                    if (JITCompiler.DEBUG) {
+                        System.err.println("no jitted code in classloader 
for file " + filename + " at class: " + className);
+                    }
+                } catch (InstantiationException ie) {
+                    if (JITCompiler.DEBUG) {
+                        System.err.println("jitted code could not be 
instantiated for file " + filename + " at class: " + className);
+                    }
+                } catch (IllegalAccessException iae) {
+                    if (JITCompiler.DEBUG) {
+                        System.err.println("jitted code could not be 
instantiated for file " + filename + " at class: " + className);
+                    }
+                }
+            } catch (IOException ioe) {
+                // TODO: log something?
+            }
+
+            // script was not found in cache above, so proceed to compile
+            if (script == null) {
+                Node scriptNode = parseFile(readStream, filename, null);
+
+                script = tryCompile(scriptNode, className, new 
JRubyClassLoader(jrubyClassLoader));
+            }
             
-            Node scriptNode = parseFile(in, filename, null);
-            
-            Script script = tryCompile(scriptNode, new 
JRubyClassLoader(jrubyClassLoader));
             if (script == null) {
                 System.err.println("Error, could not compile; pass 
-J-Djruby.jit.logging.verbose=true for more details");
             }
diff --git a/src/org/jruby/compiler/JITCompiler.java 
b/src/org/jruby/compiler/JITCompiler.java
index 239ec30..fbfa1d6 100644
--- a/src/org/jruby/compiler/JITCompiler.java
+++ b/src/org/jruby/compiler/JITCompiler.java
@@ -177,6 +177,61 @@ public class JITCompiler implements JITCompilerMBean {
             return null;
         }
     }
+
+    public static String getHashForString(String str) {
+        try {
+            return getHashForBytes(str.getBytes("UTF-8"));
+        } catch (UnsupportedEncodingException uee) {
+            throw new RuntimeException(uee);
+        }
+    }
+
+    public static String getHashForBytes(byte[] bytes) {
+        try {
+            MessageDigest sha1 = MessageDigest.getInstance("SHA1");
+            sha1.update(bytes);
+            byte[] digest = sha1.digest();
+            char[] digestChars = new char[digest.length * 2];
+            for (int i = 0; i < digest.length; i++) {
+                digestChars[i * 2] = Character.forDigit(digest[i] & 0xF, 16);
+                digestChars[i * 2 + 1] = Character.forDigit((digest[i] & 
0xF0) >> 4, 16);
+            }
+            return new String(digestChars).toUpperCase(Locale.ENGLISH);
+        } catch (NoSuchAlgorithmException nsae) {
+            throw new RuntimeException(nsae);
+        }
+    }
+
+    public static void saveToCodeCache(Ruby ruby, byte[] bytecode, String 
packageName, File cachedClassFile) {
+        String codeCache = RubyInstanceConfig.JIT_CODE_CACHE;
+        File codeCacheDir = new File(codeCache);
+        if (!codeCacheDir.exists()) {
+            ruby.getWarnings().warn("jruby.jit.codeCache directory " + 
codeCacheDir + " does not exist");
+        } else if (!codeCacheDir.isDirectory()) {
+            ruby.getWarnings().warn("jruby.jit.codeCache directory " + 
codeCacheDir + " is not a directory");
+        } else if (!codeCacheDir.canWrite()) {
+            ruby.getWarnings().warn("jruby.jit.codeCache directory " + 
codeCacheDir + " is not writable");
+        } else {
+            if (!new File(codeCache, packageName).isDirectory()) {
+                boolean createdDirs = new File(codeCache, 
packageName).mkdirs();
+                if (!createdDirs) {
+                    ruby.getWarnings().warn("could not create JIT cache dir: 
" + new File(codeCache, packageName));
+                }
+            }
+            // write to code cache
+            FileOutputStream fos = null;
+            try {
+                if (DEBUG) System.err.println("writing jitted code to to " + 
cachedClassFile);
+                fos = new FileOutputStream(cachedClassFile);
+                fos.write(bytecode);
+            } catch (Exception e) {
+                e.printStackTrace();
+                // ignore
+            } finally {
+                try {fos.close();} catch (Exception e) {}
+            }
+        }
+    }
     
     public class JITClassGenerator implements ClassCache.ClassGenerator {
         private StandardASMCompiler asmCompiler;
@@ -196,23 +251,7 @@ public class JITCompiler implements JITCompilerMBean {
         
         public JITClassGenerator(String name, String key, Ruby ruby, 
DefaultMethod method, ThreadContext context) {
             this.packageName = "ruby/jit";
-            try {
-                MessageDigest sha1 = MessageDigest.getInstance("SHA1");
-                try {
-                    sha1.update(key.getBytes("UTF-8"));
-                } catch (UnsupportedEncodingException uee) {
-                    throw new RuntimeException("fatal error: missing 
encoding UTF-8", uee);
-                }
-                byte[] digest = sha1.digest();
-                char[] digestChars = new char[digest.length * 2];
-                for (int i = 0; i < digest.length; i++) {
-                    digestChars[i * 2] = Character.forDigit(digest[i] & 0xF, 
16);
-                    digestChars[i * 2 + 1] = Character.forDigit((digest[i] & 
0xF0) >> 4, 16);
-                }
-                this.digestString = new 
String(digestChars).toUpperCase(Locale.ENGLISH);
-            } catch (NoSuchAlgorithmException nsae) {
-                throw new NotCompilableException(nsae.getLocalizedMessage());
-            }
+            this.digestString = getHashForString(key);
             this.className = packageName + "/" + 
JavaNameMangler.mangleStringForCleanJavaIdentifier(name) + "_" + digestString;
             this.name = className.replaceAll("/", ".");
             this.bodyNode = method.getBodyNode();
@@ -304,33 +343,7 @@ public class JITCompiler implements JITCompilerMBean {
             }
 
             if (codeCache != null) {
-                File codeCacheDir = new File(codeCache);
-                if (!codeCacheDir.exists()) {
-                    ruby.getWarnings().warn("jruby.jit.codeCache directory " 
+ codeCacheDir + " does not exist");
-                } else if (!codeCacheDir.isDirectory()) {
-                    ruby.getWarnings().warn("jruby.jit.codeCache directory " 
+ codeCacheDir + " is not a directory");
-                } else if (!codeCacheDir.canWrite()) {
-                    ruby.getWarnings().warn("jruby.jit.codeCache directory " 
+ codeCacheDir + " is not writable");
-                } else {
-                    if (!new File(codeCache, packageName).isDirectory()) {
-                        boolean createdDirs = new File(codeCache, 
packageName).mkdirs();
-                        if (!createdDirs) {
-                            ruby.getWarnings().warn("could not create JIT 
cache dir: " + new File(codeCache, packageName));
-                        }
-                    }
-                    // write to code cache
-                    FileOutputStream fos = null;
-                    try {
-                        if (DEBUG) System.err.println("writing jitted code 
to to " + cachedClassFile);
-                        fos = new FileOutputStream(cachedClassFile);
-                        fos.write(bytecode);
-                    } catch (Exception e) {
-                        e.printStackTrace();
-                        // ignore
-                    } finally {
-                        try {fos.close();} catch (Exception e) {}
-                    }
-                }
+                JITCompiler.saveToCodeCache(ruby, bytecode, packageName, 
cachedClassFile);
             }
             
             compiledCount.incrementAndGet();