hdf5serie  2.0.0
HDF5 Serie
file.h
1 /* Copyright (C) 2009 Markus Friedrich
2  *
3  * This library is free software; you can redistribute it and/or
4  * modify it under the terms of the GNU Lesser General Public
5  * License as published by the Free Software Foundation; either
6  * version 2.1 of the License, or (at your option) any later version.
7  *
8  * This library is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11  * Lesser General Public License for more details.
12  *
13  * You should have received a copy of the GNU Lesser General Public
14  * License along with this library; if not, write to the Free Software
15  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16  *
17  * Contact:
18  * friedrich.at.gc@googlemail.com
19  *
20  */
21 
22 #ifndef _HDF5SERIE_FILE_H_
23 #define _HDF5SERIE_FILE_H_
24 
25 #include <hdf5serie/group.h>
26 #include <boost/filesystem.hpp>
27 #include <boost/interprocess/sync/interprocess_mutex.hpp>
28 #include <boost/interprocess/sync/scoped_lock.hpp>
29 #ifdef _WIN32
30  #include <boost/interprocess/windows_shared_memory.hpp>
31 #else
32  #include <boost/interprocess/shared_memory_object.hpp>
33 #endif
34 #include <boost/interprocess/mapped_region.hpp>
35 #include <boost/container/static_vector.hpp>
36 #include <boost/uuid/uuid.hpp>
37 #include <boost/date_time/posix_time/posix_time.hpp>
38 #include <thread>
39 #include <boost/thread/thread.hpp>
40 
41 namespace H5 {
42 
43  namespace Internal {
44 #ifdef _WIN32
45  using SharedMemory = boost::interprocess::windows_shared_memory;
46  inline void SharedMemoryRemove(const char* shmName) {
47  }
48  inline SharedMemory SharedMemoryCreate(const char *shmName, boost::interprocess::mode_t mode,
49  size_t size) {
50  auto shm=SharedMemory(boost::interprocess::create_only, shmName, mode, size);
51  return shm;
52  }
53 #else
54  using SharedMemory = boost::interprocess::shared_memory_object;
55  inline void SharedMemoryRemove(const char* shmName) {
56  SharedMemory::remove(shmName);
57  }
58  inline SharedMemory SharedMemoryCreate(const char *shmName, boost::interprocess::mode_t mode,
59  size_t size) {
60  auto shm=SharedMemory(boost::interprocess::create_only, shmName, mode);
61  shm.truncate(size);
62  return shm;
63  }
64 #endif
65 
66  class ScopedLock;
67 
68  // A simple robust condition variable.
69  // The robustness is simply achived by polling in constant time intervalls instead of putting thread to sleep and wakeup.
70  // This polling delay is no problem in this scope since its only used for user visible things.
71  // So a polling delay of 1/25 second (25 frames per second) is used which equal the human visible reaction time.
72  // The interface equals boost::interprocess::interprocess_condition but not with all member functions.
73  // However, only N threads (which may be from different processes) can be waiting.
74  // This class must be implemented in a address-free way since it is placed usually in shared memory.
75  // The implementation is similar to
76  // https://cseweb.ucsd.edu/classes/sp17/cse120-a/applications/ln/lecture7.html
77  template<int N>
79  public:
80  void wait(boost::interprocess::scoped_lock<boost::interprocess::interprocess_mutex> &externLock,
81  const std::function<bool()> &pred);
82  void notify_all();
83  private:
84  boost::container::static_vector<boost::uuids::uuid, N> waiter;
85  boost::interprocess::interprocess_mutex waiterMutex;
86  };
87  }
88 
89  class Dataset;
90 
91  /* A wrapper around a HDF5 file.
92  * It handles automatically HDF5 SWMR access using a shared memory based inter-process communication:
93  * This ensure that:
94  * - only one writer is active (more writers are blocked until the writer closes the file)
95  * - readers are blocked until no writer is active or the writer is in SWMR mode
96  * - if readers are active and a writer wants to start writing the readers are notified by a reopen request.
97  */
98  class File : public GroupBase {
99  friend class Internal::ScopedLock;
100  public:
101  enum FileAccess {
104  };
114  File(const boost::filesystem::path &filename_, FileAccess type_,
115  const std::function<void()> &closeRequestCallback_=std::function<void()>(),
116  const std::function<void()> &refreshCallback_=std::function<void()>());
118  ~File() override;
121  void enableSWMR() override;
122 
123  static int getDefaultCompression() { return defaultCompression; }
124  static void setDefaultCompression(int comp) { defaultCompression=comp; }
125  static int getDefaultChunkSize() { return defaultChunkSize; }
126  static void setDefaultChunkSize(int chunk) { defaultChunkSize=chunk; }
127 
129  void refresh() override;
132  void requestFlush();
136  void flushIfRequested();
137 
140  static void dumpSharedMemory(const boost::filesystem::path &filename);
142  static void removeSharedMemory(const boost::filesystem::path &filename);
143 
144  private:
145  static int defaultCompression;
146  static int defaultChunkSize;
147 
148  void close() override;
149 
151  const boost::filesystem::path filename;
155  const std::function<void()> closeRequestCallback;
157  const std::function<void()> refreshCallback;
158 
160  std::string shmName;
162  const boost::uuids::uuid processUUID;
163 
165  enum class WriterState {
166  none, //<! no writer is currently active
167  writeRequest, //<! a writer wants to write the file but readers still exists
168  active, //<! a writer exists and is currently in dateset/attribute creation mode
169  swmr, //<! a writer exists and is currently in SWMR mode (readers can use the file using SWMR)
170  };
172  constexpr static size_t MAXREADERS { 100 };
174  constexpr static size_t MAXWRITERS { 10 };
176  struct ProcessInfo {
177  boost::uuids::uuid processUUID;
178  boost::posix_time::ptime lastAliveTime;
180  };
185  // the following member is only used for life-time handling of the shared memroy object itself
186  size_t shmUseCount { 0 }; //<! the number users of this shared memory object
187  // the following members are used to synchronize the writer and all readers.
188  boost::interprocess::interprocess_mutex mutex; //<! mutex for synchronization handling.
189  Internal::ConditionVariable<MAXREADERS+MAXWRITERS> cond; //<! a condition variable for signaling state changes.
190  // the following members represent the state of the writer and readers
191  // after setting any of these variables sharedData->cond.notify_all() must be called to notify all waiting process about the change
192  WriterState writerState { WriterState::none }; //<! the current state of the write of this file.
193  size_t activeReaders { 0 }; //<! the number of active readers on this file.
194  // the follwing members are only used for still-alive/crash detection handling
195  boost::container::static_vector<ProcessInfo, MAXREADERS+1> processes; //<! a list of all processes accessing the shared memory
196  // the follwing members are only used for flush/refresh handling
197  bool flushRequest { false }; //<! Is set to true by reader if a flush of the writer should be done. The writer resets to false after a flush.
198  };
199 
202  Internal::SharedMemory shm;
205  boost::interprocess::mapped_region region;
206 
209 
211  bool flushRequested { false };
213  WriterState lastWriterState { WriterState::none };
214 
216  static std::string createShmName(const boost::filesystem::path &filename);
217 
219  void openReader();
221  void closeReader();
223  void openWriter();
225  void closeWriter();
226 
231  void wait(Internal::ScopedLock &lock,
232  std::string_view blockingMsg, const std::function<bool()> &pred);
233 
234  // still alive pings use boost::thread since interruption point of boost, which are not availabe for std::thread, are used
235  boost::thread stillAlivePingThread; // the thread for still alive pings
236  void stillAlivePing(); // the worker function
237 
243  void listenForRequest();
245  bool exitThread { false }; // access is object is guarded by sharedData->mutex (interprocess wide)
246 
248  void initProcessInfo();
250  void deinitProcessInfo();
251 
253  void openOrCreateShm();
254  };
255 }
256 
257 #endif
bool flushRequested
True if this reader has requested a flush.
Definition: file.h:211
FileAccess
Definition: file.h:101
std::thread listenForRequestThread
Definition: file.h:241
Definition: file.h:78
static void removeSharedMemory(const boost::filesystem::path &filename)
Internal helper function which removes the shared memory associated with filename,...
Definition: file.cc:622
File(const boost::filesystem::path &filename_, FileAccess type_, const std::function< void()> &closeRequestCallback_=std::function< void()>(), const std::function< void()> &refreshCallback_=std::function< void()>())
Definition: file.cc:143
boost::uuids::uuid processUUID
a globally unique identifier for each process
Definition: file.h:177
void closeReader()
Helper function to close the file as a reader.
Definition: file.cc:411
static void dumpSharedMemory(const boost::filesystem::path &filename)
Definition: file.cc:557
void wait(Internal::ScopedLock &lock, std::string_view blockingMsg, const std::function< bool()> &pred)
Definition: file.cc:494
void enableSWMR() override
Definition: file.cc:476
void openWriter()
Helper function to open the file as a writer.
Definition: file.cc:221
static std::string createShmName(const boost::filesystem::path &filename)
transform filename to a valid boost interprocess name.
Definition: file.cc:608
~File() override
Closes the HDF5 file.
Definition: file.cc:353
open file for writing
Definition: file.h:103
void flushIfRequested()
Definition: file.cc:451
const std::function< void()> refreshCallback
This callback is called when a writer has flushed.
Definition: file.h:157
void requestFlush()
Definition: file.cc:440
Internal::SharedMemory shm
Definition: file.h:202
const std::function< void()> closeRequestCallback
This callback is called when a writer requested a close of all readers.
Definition: file.h:155
Definition: group.h:31
Definition: interface.h:273
SharedMemObject * sharedData
Pointer to the shared memory object.
Definition: file.h:208
void initProcessInfo()
Write process information of this process to the shared memory.
Definition: file.cc:260
bool exitThread
Flag which is set to true to enforce the thread to exit (on the next condition notify signal)
Definition: file.h:245
FileAccess type
the type of the process read or write
Definition: file.h:179
WriterState
A writer can be in the following state:
Definition: file.h:165
Definition: file.h:98
static constexpr size_t MAXREADERS
the maximal number of readers which can access the file simultanously
Definition: file.h:172
Definition: file.cc:46
std::string shmName
Name of the shared memory.
Definition: file.h:160
void closeWriter()
Helper function to close the file as a writer.
Definition: file.cc:395
void openReader()
Helper function to open the file as a reader.
Definition: file.cc:325
void refresh() override
Refresh the dataset of a reader.
Definition: file.cc:433
Information about a process accessing the shared memory (a process means here an instance of a File c...
Definition: file.h:176
const boost::filesystem::path filename
The name of the file.
Definition: file.h:151
WriterState lastWriterState
The last wrtierState known by this object.
Definition: file.h:213
void deinitProcessInfo()
Remove process information of this process from the shared memory.
Definition: file.cc:272
const FileAccess type
Flag if this instance is a writer or reader.
Definition: file.h:153
boost::interprocess::mapped_region region
Definition: file.h:205
Definition: file.h:184
boost::posix_time::ptime lastAliveTime
the last still alive timestamp of the process
Definition: file.h:178
static constexpr size_t MAXWRITERS
the maximal number of writers. Just need since only a fixed amount of readers+writers can wait for th...
Definition: file.h:174
void openOrCreateShm()
open or create the shared memory atomically (process with using file lock)
Definition: file.cc:173
open file for reading
Definition: file.h:102
void listenForRequest()
The worker function for the thread listenForRequestThread.
Definition: file.cc:507