Orcus
parser_token_buffer.hpp
1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2 /*
3  * This Source Code Form is subject to the terms of the Mozilla Public
4  * License, v. 2.0. If a copy of the MPL was not distributed with this
5  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
6  */
7 
8 #ifndef INCLUDED_ORCUS_DETAIL_THREAD_PARSER_TOKEN_BUFFER_HPP
9 #define INCLUDED_ORCUS_DETAIL_THREAD_PARSER_TOKEN_BUFFER_HPP
10 
11 #include "orcus/exception.hpp"
12 
13 #include <mutex>
14 #include <condition_variable>
15 
16 namespace orcus { namespace detail { namespace thread {
17 
22 template<typename _TokensT>
24 {
25  enum class state_type { parsing_progress, parsing_ended, parsing_aborted };
26 
27  typedef _TokensT tokens_type;
28 
29  mutable std::mutex m_mtx_tokens;
30  std::condition_variable m_cv_tokens_empty;
31  std::condition_variable m_cv_tokens_ready;
32 
33  tokens_type m_tokens; // token buffer used to hand over tokens to the client.
34 
35  size_t m_token_size_threshold;
36  const size_t m_max_token_size;
37 
38  state_type m_state;
39 
40  bool tokens_empty() const
41  {
42  std::lock_guard<std::mutex> lock(m_mtx_tokens);
43  return m_tokens.empty();
44  }
45 
52  void wait_until_tokens_empty()
53  {
54  std::unique_lock<std::mutex> lock(m_mtx_tokens);
55  while (!m_tokens.empty() && m_state == state_type::parsing_progress)
56  m_cv_tokens_empty.wait(lock);
57 
58  if (m_state == state_type::parsing_aborted)
60  }
61 
62 public:
63 
64  parser_token_buffer(size_t min_token_size, size_t max_token_size) :
65  m_token_size_threshold(std::max<size_t>(min_token_size, 1)),
66  m_max_token_size(max_token_size),
67  m_state(state_type::parsing_progress)
68  {
69  if (m_token_size_threshold > m_max_token_size)
70  throw invalid_arg_error(
71  "initial token size threshold is already larger than the max token size.");
72  }
73 
82  void check_and_notify(tokens_type& parser_tokens)
83  {
84  if (parser_tokens.size() < m_token_size_threshold)
85  // Still below the threshold.
86  return;
87 
88  if (!tokens_empty())
89  {
90  if (m_token_size_threshold < (m_max_token_size/2))
91  {
92  // Double the threshold and continue to parse.
93  m_token_size_threshold *= 2;
94  return;
95  }
96 
97  // We cannot increase the threshold any more. Wait for the
98  // client to finish.
99  wait_until_tokens_empty();
100  }
101 
102  std::unique_lock<std::mutex> lock(m_mtx_tokens);
103  m_tokens.swap(parser_tokens);
104  lock.unlock();
105  m_cv_tokens_ready.notify_one();
106  }
107 
116  void notify_and_finish(tokens_type& parser_tokens)
117  {
118  // Wait until the client tokens get used up.
119  wait_until_tokens_empty();
120 
121  {
122  std::lock_guard<std::mutex> lock(m_mtx_tokens);
123  m_tokens.swap(parser_tokens);
124  m_state = state_type::parsing_ended;
125  }
126  m_cv_tokens_ready.notify_one();
127  }
128 
129  void abort()
130  {
131  {
132  std::lock_guard<std::mutex> lock(m_mtx_tokens);
133  m_tokens.clear();
134  m_state = state_type::parsing_aborted;
135  }
136  m_cv_tokens_empty.notify_one();
137  }
138 
149  bool next_tokens(tokens_type& tokens)
150  {
151  tokens.clear();
152 
153  // Wait until the parser passes a new set of tokens.
154  std::unique_lock<std::mutex> lock(m_mtx_tokens);
155  while (m_tokens.empty() && m_state == state_type::parsing_progress)
156  m_cv_tokens_ready.wait(lock);
157 
158  // Get the new tokens and notify the parser.
159  tokens.swap(m_tokens);
160  state_type parsing_progress = m_state; // Make a copy so that lock can be released safely.
161 
162  lock.unlock();
163 
164  m_cv_tokens_empty.notify_one();
165 
166  return parsing_progress == state_type::parsing_progress;
167  }
168 
175  size_t token_size_threshold() const
176  {
177  if (m_state == state_type::parsing_progress)
178  return 0;
179 
180  return m_token_size_threshold;
181  }
182 };
183 
184 }}}
185 
186 #endif
187 
188 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
Definition: exception.hpp:144
Definition: parser_token_buffer.hpp:24
void notify_and_finish(tokens_type &parser_tokens)
Definition: parser_token_buffer.hpp:116
void check_and_notify(tokens_type &parser_tokens)
Definition: parser_token_buffer.hpp:82
size_t token_size_threshold() const
Definition: parser_token_buffer.hpp:175
bool next_tokens(tokens_type &tokens)
Definition: parser_token_buffer.hpp:149
Definition: exception.hpp:34
Definition: tokens.hpp:30