input_layer.h

/************************************************************
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements.  See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership.  The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License.  You may obtain a copy of the License at
* 
*   http://www.apache.org/licenses/LICENSE-2.0
* 
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied.  See the License for the
* specific language governing permissions and limitations
* under the License.
*
*************************************************************/

#ifndef SINGA_NEURALNET_INPUT_LAYER_H_
#define SINGA_NEURALNET_INPUT_LAYER_H_

#include <string>
#include <vector>
#include "neuralnet/layer.h"
#include "utils/data_shard.h"
namespace singa {
class DataLayer: virtual public InputLayer {
 public:
  void ComputeGradient(int flag, Metric* perf) override {}
  Blob<float>* mutable_data(const Layer* layer) override {
    return nullptr;
  }
  Blob<float>* mutable_grad(const Layer* layer) override {
    return nullptr;
  }
  ConnectionType dst_layer_connection() const override {
    return kOneToMany;
  }
  inline int batchsize() const { return batchsize_; }
  virtual const Record& sample() const {
    return sample_;
  }
  virtual const std::vector<Record>& records() const {
    return records_;
  }

 protected:
  int random_skip_;
  int batchsize_;
  Record sample_;
  std::vector<Record> records_;
};
class ShardDataLayer : public DataLayer {
 public:
  ~ShardDataLayer();

  void Setup(const LayerProto& proto, int npartitions) override;
  void ComputeFeature(int flag, Metric *perf) override;

 private:
  DataShard* shard_;
};

#ifdef USE_LMDB
#include <lmdb.h>
class LMDBDataLayer : public DataLayer {
 public:
  ~LMDBDataLayer();

  void Setup(const LayerProto& proto, int npartitions) override;
  void OpenLMDB(const std::string& path);
  void ComputeFeature(int flag, Metric *perf) override;
  void ConvertCaffeDatumToRecord(const CaffeDatum& datum,
                                 SingleLabelImageRecord* record);

 private:
  MDB_env* mdb_env_;
  MDB_dbi mdb_dbi_;
  MDB_txn* mdb_txn_;
  MDB_cursor* mdb_cursor_;
  MDB_val mdb_key_, mdb_value_;
};
#endif

class ParserLayer : public InputLayer {
 public:
  void ComputeFeature(int flag, Metric* perf) override;
  void ComputeGradient(int flag, Metric* perf) override {}
  ConnectionType dst_layer_connection() const override {
    return kOneToMany;
  }
  virtual void ParseRecords(int flag, const std::vector<Record>& records,
      Blob<float>* blob) = 0;
  Blob<float>* mutable_grad(const Layer* layer) override {
    return nullptr;
  }
  const Blob<float>& grad(const Layer* from) const  override {
    CHECK(false) << "Parser layer has not gradient blob";
    return grad_;
  }
};

class LabelLayer : public ParserLayer {
 public:
  void Setup(const LayerProto& proto, int npartitions) override;
  void ParseRecords(int flag, const std::vector<Record>& records,
                    Blob<float>* blob) override;
};

class MnistLayer : public ParserLayer {
 public:
  void Setup(const LayerProto& proto, int npartitions) override;
  void ParseRecords(int flag, const std::vector<Record>& records,
                    Blob<float>* blob) override;

 protected:
  float norm_a_, norm_b_;
};
class RGBImageLayer : public ParserLayer {
 public:
  void Setup(const LayerProto& proto, int npartitions) override;
  void ParseRecords(int flag, const std::vector<Record>& records,
                    Blob<float>* blob) override;

 private:
  float scale_;
  int cropsize_;
  bool mirror_;
  Blob<float> mean_;
};
class PrefetchLayer : public Layer {
 public:
  ~PrefetchLayer();
  void ComputeFeature(int flag, Metric* perf) override;
  void ComputeGradient(int flag, Metric* perf) override {}

 protected:
  std::thread thread_;
};

}  // namespace singa

#endif  // SINGA_NEURALNET_INPUT_LAYER_H_