MultiFileBase.h

Go to the documentation of this file.

// # MultiFileBase.h: Abstract base class to combine multiple files in a single one
// # Copyright (C) 2014
// # Associated Universities, Inc. Washington DC, USA.
// #
// # This library is free software; you can redistribute it and/or modify it
// # under the terms of the GNU Library General Public License as published by
// # the Free Software Foundation; either version 2 of the License, or (at your
// # option) any later version.
// #
// # This library is distributed in the hope that it will be useful, but WITHOUT
// # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
// # License for more details.
// #
// # You should have received a copy of the GNU Library General Public License
// # along with this library; if not, write to the Free Software Foundation,
// # Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
// #
// # Correspondence concerning AIPS++ should be addressed as follows:
// #        Internet email: casa-feedback@nrao.edu.
// #        Postal address: AIPS++ Project Office
// #                        National Radio Astronomy Observatory
// #                        520 Edgemont Road
// #                        Charlottesville, VA 22903-2475 USA
 
#ifndef CASA_MULTIFILEBASE_H
#define CASA_MULTIFILEBASE_H
 
// # Includes
#include <casacore/casa/aips.h>
#include <casacore/casa/IO/ByteIO.h>
#include <casacore/casa/BasicSL/String.h>
#include <casacore/casa/ostream.h>
#include <vector>
#include <memory>
 
namespace casacore {  // # NAMESPACE CASACORE - BEGIN
 
// # Forward declaration.
class AipsIO;
class HDF5Group;
class HDF5DataSet;
 
// <summary>
// Helper class for MultiFileInfo holding a data buffer
// </summary>
// <synopsis>
// The buffer can be allocated with posix_memalign (for O_DIRECT support).
// Hence the memory must be freed using free, which makes it impossible
// to use a shared_ptr to that memory. Hence it is encapsulated in this class.
// </synopsis>
class MultiFileBuffer {
 public:
  MultiFileBuffer(size_t bufSize, Bool useODirect);
  ~MultiFileBuffer() {
    if (itsData) free(itsData);
  }
  // Forbid copy constructor.
  MultiFileBuffer(const MultiFileBuffer&) = delete;
  // Forbid assignment.
  MultiFileBuffer& operator=(const MultiFileBuffer&) = delete;
  char* data() { return itsData; }
 
 private:
  // Data members
  char* itsData;
};
 
// <summary>
// Helper class for MultiFileBase containing info per logical file.
// </summary>
// <synopsis>
// This struct defines the basic fields describing a logical file in a
// class derived from MultiFileBase (such as MultiFile or MultiHDF5).
// </synopsis>
// <use visibility=local>
struct MultiFileInfo {
  // Initialize the object. The buffer is created when the file is opened.
  explicit MultiFileInfo();
  // Allocate the buffer.
  void allocBuffer(size_t bufSize, Bool useODirect) {
    buffer = std::make_shared<MultiFileBuffer>(bufSize, useODirect);
  }
  // # Data members.
  std::vector<Int64> blockNrs;              // physical blocknrs for this logical file
  Int64 curBlock;                           // the data block held in buffer (<0 is none)
  Int64 fsize;                              // file size (in bytes)
  String name;                              // the virtual file name
  Bool nested;                              // is the file a nested MultiFile?
  Bool dirty;                               // has data in buffer been changed?
  std::shared_ptr<MultiFileBuffer> buffer;  // buffer holding a data block
  std::shared_ptr<HDF5Group> group;
  std::shared_ptr<HDF5DataSet> dataSet;
};
ostream& operator<<(ostream&, const MultiFileInfo&);
AipsIO& operator<<(AipsIO&, const MultiFileInfo&);
AipsIO& operator>>(AipsIO&, MultiFileInfo&);
void getInfoVersion1(AipsIO&, std::vector<MultiFileInfo>&);
 
// <summary>
// Abstract base class to combine multiple logical files in a single one.
// </summary>
 
// <use visibility=export>
 
// <reviewed reviewer="" date="" tests="tMultiFile" demos="">
// </reviewed>
 
// <synopsis>
// This class is the abstract base class for classes defining a container
// file holding multiple logical files. These classes are meant as a
// container files for the storage manager files of a table to reduce the
// number of files used (especially for Lustre) and to reduce the number
// of open files (especially when concatenating tables).
// The derived classes MultiFile and MultiHDF5 implement such container
// files using a regular file and HDF5, resp.
//
// MultiFileBase implements several functions with common functionality
// for the derived classes.
//
// A logical file is represented by an MFFileIO object, which is derived
// from ByteIO and as such part of the casacore IO framework. It makes it
// possible for applications to access a logical file in the same way as
// a regular file.
// </synopsis>
 
class MultiFileBase {
 public:
  // Create a MultiFileBase object with the given name.
  // <br>Upon creation of the container file the block size can be given.
  // If <=0, it uses the block size of the file system the file is on,
  // but it will not be less than the absolute value of the given block size.
  // <br>If useODIrect=True, it means that O_DIRECT is used. If the OS does not
  // support it (as determined at configure time), the flag will always be
  // set to False. If True, the data buffers will have a proper alignment
  // and size (as needed by O_DIRECT).
  MultiFileBase(const String& name, Int blockSize, Bool useODirect);
 
  // The destructor flushes dirty blocks and closes the container file.
  virtual ~MultiFileBase();
 
  // Forbid copy constructor.
  MultiFileBase(const MultiFileBase&) = delete;
 
  // Forbid assignment.
  MultiFileBase& operator=(const MultiFileBase&) = delete;
 
  // Open the correct MultiFileBase (as plain or HDF5).
  static std::shared_ptr<MultiFileBase> openMF(const String& fileName);
 
  // Make the correct MultiFileBase object for a nested file.
  virtual std::shared_ptr<MultiFileBase> makeNested(const std::shared_ptr<MultiFileBase>& parent,
                                                    const String& name, ByteIO::OpenOption,
                                                    Int blockSize) const = 0;
 
  // Get the file name of the MultiFileBase container file.
  String fileName() const { return itsName; }
 
  // Is the container file writable?
  Bool isWritable() const { return itsWritable; }
 
  // Open the given logical file and return its file id.
  // If the name is unknown, an exception is thrown.
  // It allocates the internal buffer of the logical file.
  Int openFile(const String& name);
 
  // Create a new logical file and return its file id.
  // Only the base name of the given file name is used. In this way the
  // MultiFileBase container file can be moved.
  // If the logical file already exists, it is deleted if ByteIO::New is
  // given. Otherwise an exception is thrown.
  // It allocates the internal buffer of the logical file.
  Int createFile(const String& name, ByteIO::OpenOption = ByteIO::New);
 
  // Flush the possible dirty buffer of the given logical file.
  void flushFile(Int fileId);
 
  // Close a logical file.
  // It flushes and deallocates its buffer.
  void closeFile(Int fileId);
 
  // Delete a logical file. It adds its blocks to the free block list.
  void deleteFile(Int fileId);
 
  // Get the size of a logical file.
  Int64 fileSize(Int fileId) const;
 
  // Read a block at the given offset in the logical file.
  // It returns the actual size read.
  Int64 read(Int fileId, void* buffer, Int64 size, Int64 offset);
 
  // Write a block at the given offset in the logical file.
  // It returns the actual size written.
  Int64 write(Int fileId, const void* buffer, Int64 size, Int64 offset);
 
  // Truncate the logical file to the given size.
  void truncate(Int fileId, Int64 size);
 
  // Reopen the underlying file for read/write access.
  // Nothing will be done if the file is writable already.
  // Otherwise it will be reopened and an exception will be thrown
  // if it is not possible to reopen it for read/write access.
  virtual void reopenRW() = 0;
 
  // Flush the file by writing all dirty data and all header info.
  void flush();
 
  // Get the block size used.
  Int64 blockSize() const { return itsBlockSize; }
 
  // Get the nr of logical files.
  uInt nfile() const;
 
  // Get the total nr of data blocks used.
  Int64 nblock() const { return itsNrBlock; }
 
  // Get the info object (for test purposes mainly).
  const std::vector<MultiFileInfo>& info() const { return itsInfo; }
 
  // Get the free blocks (for test purposes mainly).
  const std::vector<Int64>& freeBlocks() const { return itsFreeBlocks; }
 
  // Return the file id of a file in the MultiFileBase object.
  // If the name is unknown, an exception is thrown if throwExcp is set.
  // Otherwise it returns -1.
  Int fileId(const String& name, Bool throwExcp = True) const;
 
  // Is O_DIRECT used?
  Bool useODirect() const { return itsUseODirect; }
 
 protected:
  // Resync with another process by clearing the buffers and rereading
  // the header. The header is only read if its counter has changed.
  void resync();
 
  // Fsync the file (i.e., force the data to be physically written).
  virtual void fsync() = 0;
 
 private:
  // Write the dirty block and clear dirty flag.
  void writeDirty(MultiFileInfo& info) {
    writeBlock(info, info.curBlock, info.buffer->data());
    info.dirty = False;
  }
 
  // Add a file to the MultiFileBase object. It returns the file id.
  // Only the base name of the given file name is used. In this way the
  // MultiFileBase container file can be moved.
  Int addFile(const String& name);
 
  // Do the class-specific actions on opening a logical file.
  virtual void doOpenFile(MultiFileInfo&) = 0;
  // Do the class-specific actions on closing a logical file.
  virtual void doCloseFile(MultiFileInfo&) = 0;
  // Do the class-specific actions on adding a logical file.
  virtual void doAddFile(MultiFileInfo&) = 0;
  // Do the class-specific actions on deleting a logical file.
  virtual void doDeleteFile(MultiFileInfo&) = 0;
  // Truncate the container file to <src>nrblk</src> blocks.
  virtual void doTruncateFile(MultiFileInfo& info, uInt64 nrblk) = 0;
  // Flush the container file.
  virtual void doFlushFile() = 0;
  // Flush and close the container file.
  virtual void close() = 0;
  // Write the header info.
  virtual void writeHeader() = 0;
  // Read the header info. If always==False, the info is only read if the
  // header counter has changed.
  virtual void readHeader(Bool always = True) = 0;
  // Extend a logical file to fit lastblk.
  virtual void extend(MultiFileInfo& info, Int64 lastblk) = 0;
  // Write a data block of a logical file into the container file.
  virtual void writeBlock(MultiFileInfo& info, Int64 blknr, const void* buffer) = 0;
  // Read a data block of a logical file from the container file.
  virtual void readBlock(MultiFileInfo& info, Int64 blknr, void* buffer) = 0;
 
 protected:
  // Set the flags and blockSize for a new MultiFile/HDF5.
  void setNewFile();
 
  // # Data members
  String itsName;
  Int64 itsBlockSize;   // The blocksize used
  Int64 itsNrBlock;     // The total nr of blocks actually used
  Int64 itsHdrCounter;  // Counter of header changes
  std::vector<MultiFileInfo> itsInfo;
  std::shared_ptr<MultiFileBuffer> itsBuffer;
  Bool itsUseODirect;  // use O_DIRECT?
  Bool itsWritable;    // Is the file writable?
  Bool itsChanged;     // Has header info changed since last flush?
  std::vector<Int64> itsFreeBlocks;
};
 
}  // namespace casacore
 
#endif