Apache SINGA
A distributed deep learning platform .
 All Classes Namespaces Files Functions Variables Typedefs Enumerator Macros
tensor_io.h
Go to the documentation of this file.
1 #ifndef MSHADOW_TENSOR_IO_H
2 #define MSHADOW_TENSOR_IO_H
3 
8 #include <cstdio>
9 #include "tensor.h"
10 
11 namespace mshadow{
12  namespace utils{
18  class IStream{
19  public:
26  virtual size_t Read( void *ptr, size_t size ) = 0;
32  virtual void Write( const void *ptr, size_t size ) = 0;
34  virtual ~IStream( void ){}
35  };
36  };
37 
45  template<int dim,typename TStream>
46  inline void SaveBinary( TStream &fo, const Tensor<cpu,dim> &src );
48  template<int dim,typename TStream>
49  inline void SaveBinary( TStream &fo, const Tensor<gpu,dim> &src );
50 
61  template<int dim,typename TStream>
62  inline void LoadBinary( TStream &fi, Tensor<cpu,dim> &dst, bool pre_alloc );
64  template<int dim,typename TStream>
65  inline void LoadBinary( TStream &fi, Tensor<gpu,dim> &dst, bool pre_alloc );
66 
67  namespace utils{
69  class FileStream: public IStream{
70  public:
72  FileStream( FILE *fp ):fp_(fp){}
73  virtual size_t Read( void *ptr, size_t size ){
74  return fread( ptr, size, 1, fp_ );
75  }
76  virtual void Write( const void *ptr, size_t size ){
77  fwrite( ptr, size, 1, fp_ );
78  }
80  inline void Close( void ){
81  fclose( fp_ );
82  }
83  private:
84  FILE *fp_;
85  };
86  };
87 };
88 
89 namespace mshadow{
90  // implementations
91  template<int dim, typename TStream>
92  inline void SaveBinary( TStream &fo, const Tensor<cpu,dim> &src_ ){
93  fo.Write( src_.shape.shape_, sizeof(index_t) * dim );
94  Tensor<cpu,2> src = src_.FlatTo2D();
95  for( index_t i = 0; i < src.shape[1]; ++ i ){
96  fo.Write( src[i].dptr, sizeof(real_t)*src.shape[0] );
97  }
98  }
99  template<int dim, typename TStream>
100  inline void SaveBinary( TStream &fo, const Tensor<gpu,dim> &src ){
101  // copy to CPU, then save
102  Tensor<cpu,dim> tmp( src.shape );
103  AllocSpace( tmp );
104  Copy( tmp, src );
105  SaveBinary( fo, tmp );
106  FreeSpace( tmp );
107  }
108 
109  template<int dim, typename TStream>
110  inline void LoadBinary( TStream &fi, Tensor<cpu,dim> &dst_, bool pre_alloc ){
111  Shape<dim> shape;
112  utils::Assert( fi.Read( shape.shape_, sizeof(index_t) * dim ) != 0, "mshadow::LoadBinary" );
113  if( pre_alloc ){
114  utils::Assert( shape == dst_.shape );
115  }else{
116  dst_.shape = shape; AllocSpace( dst_ );
117  }
118  Tensor<cpu,2> dst = dst_.FlatTo2D();
119  if( dst.shape[0] == 0 ) return;
120  for( index_t i = 0; i < dst.shape[1]; ++ i ){
121  utils::Assert( fi.Read( dst[i].dptr, sizeof(real_t)*dst.shape[0] ) != 0, "mshadow::LoadBinary" );
122  }
123  }
124  template<int dim, typename TStream>
125  inline void LoadBinary( TStream &fi, Tensor<gpu,dim> &dst, bool pre_alloc ){
126  Tensor<cpu,dim> tmp;
127  LoadBinary( fi, tmp, false );
128  if( pre_alloc ){
129  utils::Assert( tmp.shape == dst.shape );
130  }else{
131  dst.shape = tmp.shape; AllocSpace( dst );
132  }
133  Copy( dst, tmp );
134  FreeSpace( tmp );
135  }
136 };
137 #endif // TENSOR_IO_H
unsigned index_t
type that will be used for index
Definition: tensor_base.h:123
virtual size_t Read(void *ptr, size_t size)
read data from stream
Definition: tensor_io.h:73
virtual size_t Read(void *ptr, size_t size)=0
read data from stream
void FreeSpace(Tensor< cpu, dim > &obj)
CPU/GPU: free the space of tensor, will set obj.dptr to NULL.
Definition: tensor_cpu-inl.hpp:36
void Assert(bool exp)
assert a expression is true
Definition: tensor_base.h:285
implementation of file i/o stream
Definition: tensor_io.h:69
index_t shape_[kMaxShape]
storing the dimension information
Definition: tensor.h:129
float real_t
type that will be used for content
Definition: tensor_base.h:118
header file of tensor data structure and functions covention: this lib requires explicit memory alloc...
FileStream(FILE *fp)
constructor
Definition: tensor_io.h:72
void Close(void)
close file
Definition: tensor_io.h:80
void SaveBinary(TStream &fo, const Tensor< cpu, dim > &src)
CPU/GPU: save a tensor by binary format, for GPU version, a temp Tensor<cpu,dim> storage will be allo...
Definition: tensor_io.h:92
virtual void Write(const void *ptr, size_t size)
write data to stream
Definition: tensor_io.h:76
real_t * dptr
pointer to the data
Definition: tensor.h:215
MSHADOW_XINLINE Tensor< Device, 2 > FlatTo2D(void) const
flatten the tensor to 2 dimension, collapse the higher dimensions together
Definition: tensor.h:229
void Copy(Tensor< cpu, dim > dst, const Tensor< cpu, dim > &src)
copy data from one tensor to another, with same shape
Definition: tensor_cpu-inl.hpp:42
Shape< dimension > shape
shape of the tensor
Definition: tensor.h:217
void AllocSpace(Tensor< cpu, dim > &obj, bool pad=MSHADOW_ALLOC_PAD)
CPU/CPU: allocate space for CTensor, according to the shape in the obj this function is responsible t...
Definition: tensor_cpu-inl.hpp:14
interface of stream I/O, used to serialize data, it is not restricted to only this interface in SaveB...
Definition: tensor_io.h:18
virtual ~IStream(void)
virtual destructor
Definition: tensor_io.h:34
general tensor
Definition: tensor.h:206
void LoadBinary(TStream &fi, Tensor< cpu, dim > &dst, bool pre_alloc)
CPU/GPU: load a tensor by binary format, for GPU version, a temp Tensor<cpu,dim> storage will be allo...
Definition: tensor_io.h:110
virtual void Write(const void *ptr, size_t size)=0
write data to stream