lshifr
3/6/2013 - 11:04 PM

Java code for Mathematica to read entire file of floats

Java code for Mathematica to read entire file of floats

(* Create a file *)

fname = "C:\\Temp\\randomdataReal";

n = 10^6;
outputstr = OpenWrite[fname, BinaryFormat -> True]
lreals = RandomReal[100, {n, 10}];
BinaryWrite[outputstr, lreals, {"Real32", "Real32", "Real32", 
  "Real32", "Real32", "Real32", "Real32", "Real32", "Real32", 
  "Real32"}, ByteOrdering -> +1]
Close[outputstr]

(* Read the data *)

 read = 
   EntireTableReader`getFloatTable["C:\\Temp\\randomdataReal", 10*4, 
    1000]; // AbsoluteTiming

(*  {0.3789062, Null}  *)

(* Need to partition back to columns manually *)
readpart = Partition[read, 10];

(* compare - note that the results are only approximately the same  *)

Chop[readpart - lreals, 10^-5] // Total

(* {0, 0, 0, 0, 0, 0, 0, 0, 0, 0} *)



(* Compare *)

JCompileLoad@"
        import java.io.*;
        import java.nio.ByteBuffer;
        import java.nio.MappedByteBuffer;
        import java.nio.channels.FileChannel;
        import java.nio.channels.FileChannel.MapMode;
        import java.util.Arrays;
 
        public class EntireTableReader{        
          public  static float[] getFloatTable(String filename, int rowByteCount, int rowChunkSize) 
            throws FileNotFoundException, IOException{
                File fl = new File(filename);
                FileInputStream str = new FileInputStream(fl);
                FileChannel ch = str.getChannel();
                MappedByteBuffer mb = ch.map( \
                FileChannel.MapMode.READ_ONLY, 0L, ch.size( ));
                final int buffrows = rowChunkSize;
                final int buffSize = buffrows * rowByteCount;
                byte[] buffer = new byte[buffSize];
                int rows  = (int)(fl.length()/rowByteCount);
   		          int ncols = (int)(rowByteCount/4);
                float[] result = new float[rows*ncols];
                byte[] intermediate = new byte[rowByteCount*rows];
                int cycles  = (int)(rows/buffrows);
                int remaining = rows % buffrows;
                byte[] remBuffer = new byte[remaining * rowByteCount];
                int ctr=0; 
                try{
                   for(int j=0;j<cycles;j++){
                      int bctr = 0;
                      mb.get(buffer);
                      for(int i=0;i < buffrows;i++){		
                         System.arraycopy(buffer, bctr,intermediate,ctr,rowByteCount);
 						             ctr+=rowByteCount;
                         bctr+=rowByteCount;
                      }
                   }
                   int bctr = 0;
                   mb.get(remBuffer);
                   for(int i=0; i < remaining;i++){  				
                       System.arraycopy(remBuffer, bctr,intermediate,ctr,rowByteCount);
 					             ctr+=rowByteCount;
                       bctr+=rowByteCount;
                   }
                   ByteBuffer buf2 = ByteBuffer.wrap(intermediate);
                   for(int i=0;i<rows*ncols;i++){
                       result[i]=buf2.getFloat();
                   }			
                } finally{
                   str.close();
                }
                return result;		
         }	
      }"