snap相关（个人总结）

 1 struct SnapRealm {
 2   // realm state
 3 
 4   sr_t srnode;
 5 
 6   // in-memory state
 7   MDCache *mdcache;
 8   CInode *inode;
 9 
10   bool open;                        // set to true once all past_parents are opened
11   SnapRealm *parent;
12   set<SnapRealm*> open_children;    // active children that are currently open
13   map<inodeno_t,SnapRealm*> open_past_parents;  // these are explicitly pinned.
14 
15   // cache
16   snapid_t cached_seq;           // max seq over self and all past+present parents.
17   snapid_t cached_last_created;  // max last_created over all past+present parents
18   snapid_t cached_last_destroyed;
19   set<snapid_t> cached_snaps;
20   SnapContext cached_snap_context;
21 
22   bufferlist cached_snap_trace;
23 
24   elist<CInode*> inodes_with_caps;             // for efficient realm splits
25   map<client_t, xlist<Capability*>* > client_caps;   // to identify clients who need snap notifications

  1 #include "SnapRealm.h"
  2 #include "MDCache.h"
  3 #include "MDS.h"
  4 
  5 #include "messages/MClientSnap.h"
  6 
  7 
  8 /*
  9  * SnapRealm
 10  */
 11 
 12 #define dout_subsys ceph_subsys_mds
 13 #undef dout_prefix
 14 #define dout_prefix _prefix(_dout, mdcache->mds->get_nodeid(), inode, srnode.seq, this)
 15 static ostream& _prefix(std::ostream *_dout, int whoami, CInode *inode,
 16             uint64_t seq, SnapRealm *realm) {
 17   return *_dout << " mds." << whoami
 18         << ".cache.snaprealm(" << inode->ino()
 19         << " seq " << seq << " " << realm << ") ";
 20 }
 21 
 22 ostream& operator<<(ostream& out, const SnapRealm& realm) 
 23 {
 24   out << "snaprealm(" << realm.inode->ino()
 25       << " seq " << realm.srnode.seq
 26       << " lc " << realm.srnode.last_created
 27       << " cr " << realm.srnode.created;
 28   if (realm.srnode.created != realm.srnode.current_parent_since)
 29     out << " cps " << realm.srnode.current_parent_since;
 30   out << " snaps=" << realm.srnode.snaps;
 31   if (realm.srnode.past_parents.size()) {
 32     out << " past_parents=(";
 33     for (map<snapid_t, snaplink_t>::const_iterator p = realm.srnode.past_parents.begin(); 
 34      p != realm.srnode.past_parents.end(); 
 35      ++p) {
 36       if (p != realm.srnode.past_parents.begin()) out << ",";
 37       out << p->second.first << "-" << p->first
 38       << "=" << p->second.ino;
 39     }
 40     out << ")";
 41   }
 42   out << " " << &realm << ")";
 43   return out;
 44 }
 45 
 46 //添加parent到open_past_parents的map容器中
 47 void SnapRealm::add_open_past_parent(SnapRealm *parent)
 48 {
 49   open_past_parents[parent->inode->ino()] = parent;
 50   parent->inode->get(CInode::PIN_PASTSNAPPARENT);        //标记为CInode::PIN_PASTSNAPPARENT
 51 }
 52 
 53 struct C_SR_RetryOpenParents : public MDSInternalContextBase {
 54   SnapRealm *sr;
 55   snapid_t first, last, parent_last;
 56   inodeno_t parent;
 57   MDSInternalContextBase* fin;
 58   C_SR_RetryOpenParents(SnapRealm *s, snapid_t f, snapid_t l, snapid_t pl,
 59             inodeno_t p, MDSInternalContextBase *c) :
 60     sr(s), first(f), last(l), parent_last(pl),  parent(p), fin(c) {}
 61   MDS *get_mds() { return sr->mdcache->mds; }
 62   void finish(int r) {
 63     if (r < 0)
 64       sr->_remove_missing_parent(parent_last, parent, r);
 65     if (sr->_open_parents(fin, first, last))
 66       fin->complete(0);
 67   }
 68 };
 69 
 70 //删除找不到snapid的parent
 71 void SnapRealm::_remove_missing_parent(snapid_t snapid, inodeno_t parent, int err)
 72 {
 73   map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.find(snapid);
 74   if (p != srnode.past_parents.end()) {
 75     dout(10) << __func__ << " " << parent << " [" << p->second.first << ","
 76          << p->first << "]  errno " << err << dendl;
 77     srnode.past_parents.erase(p);
 78   } else {
 79     dout(10) << __func__ << " " << parent << " not found" << dendl;
 80   }
 81 }
 82 
 83 //判定parents是否open，返回true or false
 84 bool SnapRealm::_open_parents(MDSInternalContextBase *finish, snapid_t first, snapid_t last)
 85 {
 86   dout(10) << "open_parents [" << first << "," << last << "]" << dendl;
 87   if (open) 
 88     return true;
 89 
 90   // make sure my current parents‘ parents are open...
 91   if (parent) {
 92     dout(10) << " current parent [" << srnode.current_parent_since << ",head] is " << *parent
 93          << " on " << *parent->inode << dendl;
 94     if (last >= srnode.current_parent_since &&
 95     !parent->_open_parents(finish, MAX(first, srnode.current_parent_since), last))
 96       return false;
 97   }
 98 
 99   // and my past parents too!
100   assert(srnode.past_parents.size() >= open_past_parents.size());
101   if (srnode.past_parents.size() > open_past_parents.size()) {
102     for (map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.begin();
103      p != srnode.past_parents.end();
104      ++p) {    
105       dout(10) << " past_parent [" << p->second.first << "," << p->first << "] is "
106            << p->second.ino << dendl;
107       CInode *parent = mdcache->get_inode(p->second.ino);
108       if (!parent) {
109     C_SR_RetryOpenParents *fin = new C_SR_RetryOpenParents(this, first, last, p->first,
110                                    p->second.ino, finish);
111     mdcache->open_ino(p->second.ino, mdcache->mds->mdsmap->get_metadata_pool(), fin);
112     return false;
113       }
114       assert(parent->snaprealm);  // hmm!
115       if (!parent->snaprealm->_open_parents(finish, p->second.first, p->first))
116     return false;
117       if (!open_past_parents.count(p->second.ino)) {
118     add_open_past_parent(parent->snaprealm);
119       }
120     }
121   }
122 
123   open = true;
124   return true;
125 }
126 
127 //判断从first到last的snap的past_parents是否open
128 bool SnapRealm::have_past_parents_open(snapid_t first, snapid_t last)    
129 {
130   dout(10) << "have_past_parents_open [" << first << "," << last << "]" << dendl;
131   if (open)
132     return true;
133 
134   for (map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.lower_bound(first);
135        p != srnode.past_parents.end();
136        ++p) {
137     if (p->second.first > last)
138       break;
139     dout(10) << " past parent [" << p->second.first << "," << p->first << "] was "
140          << p->second.ino << dendl;
141     if (open_past_parents.count(p->second.ino) == 0) {
142       dout(10) << " past parent " << p->second.ino << " is not open" << dendl;
143       return false;
144     }
145     if (!open_past_parents[p->second.ino]->have_past_parents_open(MAX(first, p->second.first),
146                                   MIN(last, p->first)))
147       return false;
148   }
149 
150   open = true;
151   return true;
152 }
153 
154 
155 void SnapRealm::close_parents()
156 {
157   for (map<inodeno_t,SnapRealm*>::iterator p = open_past_parents.begin();
158        p != open_past_parents.end();
159        ++p)
160     p->second->inode->put(CInode::PIN_PASTSNAPPARENT);   //解除标记CInode::PIN_PASTSNAPPARENT
161   open_past_parents.clear();  //清空
162 }
163 
164 
165 /*
166  * get list of snaps for this realm.  we must include parents‘ snaps
167  * for the intervals during which they were our parent.
168  */
169  //将realm中的snaps取出并存入第一个参数set容器中
170 void SnapRealm::build_snap_set(set<snapid_t> &s,
171                    snapid_t& max_seq, snapid_t& max_last_created, snapid_t& max_last_destroyed,
172                    snapid_t first, snapid_t last)
173 {
174   dout(10) << "build_snap_set [" << first << "," << last << "] on " << *this << dendl;
175 
176   if (srnode.seq > max_seq)
177     max_seq = srnode.seq;
178   if (srnode.last_created > max_last_created)
179     max_last_created = srnode.last_created;
180   if (srnode.last_destroyed > max_last_destroyed)
181     max_last_destroyed = srnode.last_destroyed;
182 
183   // include my snaps within interval [first,last]
184   for (map<snapid_t, SnapInfo>::iterator p = srnode.snaps.lower_bound(first); // first element >= first
185        p != srnode.snaps.end() && p->first <= last;
186        ++p)
187     s.insert(p->first);
188 
189   // include snaps for parents during intervals that intersect [first,last]
190   for (map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.lower_bound(first);
191        p != srnode.past_parents.end() && p->first >= first && p->second.first <= last;
192        ++p) {
193     CInode *oldparent = mdcache->get_inode(p->second.ino);
194     assert(oldparent);  // call open_parents first!
195     assert(oldparent->snaprealm);
196     oldparent->snaprealm->build_snap_set(s, max_seq, max_last_created, max_last_destroyed,
197                      MAX(first, p->second.first),
198                      MIN(last, p->first));           //递归，将past_parent的snapid加入set容器
199   }
200   if (srnode.current_parent_since <= last && parent)
201     parent->build_snap_set(s, max_seq, max_last_created, max_last_destroyed,
202                MAX(first, srnode.current_parent_since), last);         //递归，将current_parent的snapid加入set容器
203 }
204 
205 
206 //检查cache中的数据结构并更新cached_snaps和cached_snap_trace
207 void SnapRealm::check_cache()
208 {
209   if (cached_seq >= srnode.seq)
210     return;
211 
212   cached_snaps.clear();
213   cached_snap_context.clear();
214 
215   cached_last_created = srnode.last_created;
216   cached_last_destroyed = srnode.last_destroyed;
217   cached_seq = srnode.seq;
218   build_snap_set(cached_snaps, cached_seq, cached_last_created, cached_last_destroyed,
219          0, CEPH_NOSNAP);
220 
221   cached_snap_trace.clear();
222   build_snap_trace(cached_snap_trace);
223   
224   dout(10) << "check_cache rebuilt " << cached_snaps
225        << " seq " << srnode.seq
226        << " cached_seq " << cached_seq
227        << " cached_last_created " << cached_last_created
228        << " cached_last_destroyed " << cached_last_destroyed
229        << ")" << dendl;
230 }
231 
232 
233 const set<snapid_t>& SnapRealm::get_snaps()
234 {
235   check_cache();
236   dout(10) << "get_snaps " << cached_snaps
237        << " (seq " << srnode.seq << " cached_seq " << cached_seq << ")"
238        << dendl;
239   return cached_snaps;
240 }
241 
242 /*
243  * build vector in reverse sorted order
244  */
245  //循环将存储snapid的set中的内容放入返回值cached_snap_context中
246 const SnapContext& SnapRealm::get_snap_context()
247 {
248   check_cache();
249 
250   if (!cached_snap_context.seq) {
251     cached_snap_context.seq = cached_seq;
252     cached_snap_context.snaps.resize(cached_snaps.size());
253     unsigned i = 0;
254     for (set<snapid_t>::reverse_iterator p = cached_snaps.rbegin();
255      p != cached_snaps.rend();
256      ++p)
257       cached_snap_context.snaps[i++] = *p;
258   }
259 
260   return cached_snap_context;
261 }
262 
263 //得到cache_snap，并将其info存入infomap的map容器中
264 void SnapRealm::get_snap_info(map<snapid_t,SnapInfo*>& infomap, snapid_t first, snapid_t last)
265 {
266   const set<snapid_t>& snaps = get_snaps();
267   dout(10) << "get_snap_info snaps " << snaps << dendl;
268 
269   // include my snaps within interval [first,last]
270   for (map<snapid_t, SnapInfo>::iterator p = srnode.snaps.lower_bound(first); // first element >= first
271        p != srnode.snaps.end() && p->first <= last;
272        ++p)
273     infomap[p->first] = &p->second;    //令键为p->first的infomap的值等于&p->second
274 
275   // include snaps for parents during intervals that intersect [first,last]
276   for (map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.lower_bound(first);
277        p != srnode.past_parents.end() && p->first >= first && p->second.first <= last;
278        ++p) {
279     CInode *oldparent = mdcache->get_inode(p->second.ino);
280     assert(oldparent);  // call open_parents first!
281     assert(oldparent->snaprealm);
282     oldparent->snaprealm->get_snap_info(infomap,
283                     MAX(first, p->second.first),
284                     MIN(last, p->first));
285   }
286   if (srnode.current_parent_since <= last && parent)
287     parent->get_snap_info(infomap, MAX(first, srnode.current_parent_since), last);
288 }
289 
290 const string& SnapRealm::get_snapname(snapid_t snapid, inodeno_t atino)
291 {
292   if (srnode.snaps.count(snapid)) {
293     if (atino == inode->ino())
294       return srnode.snaps[snapid].name;
295     else
296       return srnode.snaps[snapid].get_long_name();
297   }
298 
299   map<snapid_t,snaplink_t>::iterator p = srnode.past_parents.lower_bound(snapid);
300   if (p != srnode.past_parents.end() && p->second.first <= snapid) {
301     CInode *oldparent = mdcache->get_inode(p->second.ino);
302     assert(oldparent);  // call open_parents first!
303     assert(oldparent->snaprealm);    
304     return oldparent->snaprealm->get_snapname(snapid, atino);
305   }
306 
307   assert(srnode.current_parent_since <= snapid);
308   assert(parent);
309   return parent->get_snapname(snapid, atino);
310 }
311 
312 //判断第一个参数n，在srnode.snaps从first到last范围内找出对应的snapid
313 snapid_t SnapRealm::resolve_snapname(const string& n, inodeno_t atino, snapid_t first, snapid_t last)
314 {
315   // first try me
316   dout(10) << "resolve_snapname ‘" << n << "‘ in [" << first << "," << last << "]" << dendl;
317 
318   //snapid_t num;
319   //if (n[0] == ‘~‘) num = atoll(n.c_str()+1);
320 
321   bool actual = (atino == inode->ino());
322   string pname;
323   inodeno_t pino;
324   if (!actual) {
325     if (!n.length() ||
326     n[0] != ‘_‘) return 0;
327     int next_ = n.find(‘_‘, 1);
328     if (next_ < 0) return 0;
329     pname = n.substr(1, next_ - 1);
330     pino = atoll(n.c_str() + next_ + 1);
331     dout(10) << " " << n << " parses to name ‘" << pname << "‘ dirino " << pino << dendl;
332   }
333 
334 //根据snapinfo得到snapid
335   for (map<snapid_t, SnapInfo>::iterator p = srnode.snaps.lower_bound(first); // first element >= first
336        p != srnode.snaps.end() && p->first <= last;
337        ++p) {
338     dout(15) << " ? " << p->second << dendl;
339     //if (num && p->second.snapid == num)
340     //return p->first;
341     if (actual && p->second.name == n)
342     return p->first;
343     if (!actual && p->second.name == pname && p->second.ino == pino)
344       return p->first;
345   }
346 
347     // include snaps for parents during intervals that intersect [first,last]
348   for (map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.lower_bound(first);
349        p != srnode.past_parents.end() && p->first >= first && p->second.first <= last;
350        ++p) {
351     CInode *oldparent = mdcache->get_inode(p->second.ino);
352     assert(oldparent);  // call open_parents first!
353     assert(oldparent->snaprealm);
354     snapid_t r = oldparent->snaprealm->resolve_snapname(n, atino,
355                             MAX(first, p->second.first),
356                             MIN(last, p->first));
357     if (r)
358       return r;
359   }
360   if (parent && srnode.current_parent_since <= last)
361     return parent->resolve_snapname(n, atino, MAX(first, srnode.current_parent_since), last);
362   return 0;
363 }
364 
365 
366 //调整parent令变量parent等于当前parent
367 void SnapRealm::adjust_parent()
368 {
369   SnapRealm *newparent = inode->get_parent_dn()->get_dir()->get_inode()->find_snaprealm();
370   if (newparent != parent) {
371     dout(10) << "adjust_parent " << parent << " -> " << newparent << dendl;
372     if (parent)
373       parent->open_children.erase(this);
374     parent = newparent;
375     if (parent)
376       parent->open_children.insert(this);
377     
378     invalidate_cached_snaps();
379   }
380 }
381 
382 //将child分离出当前realm
383 void SnapRealm::split_at(SnapRealm *child)
384 {
385   dout(10) << "split_at " << *child 
386        << " on " << *child->inode << dendl;
387 
388   if (inode->is_mdsdir() || !child->inode->is_dir()) {
389     // it‘s not a dir.
390     if (child->inode->containing_realm) {
391       //  - no open children.
392       //  - only need to move this child‘s inode‘s caps.
393       child->inode->move_to_realm(child);           //脱离containing_realm，加入到child realm
394     } else {
395       // no caps, nothing to move/split.
396       dout(20) << " split no-op, no caps to move on file " << *child->inode << dendl;
397       assert(!child->inode->is_any_caps());
398     }
399     return;
400   }
401 
402   // it‘s a dir.
403 
404   // split open_children
405   dout(10) << " open_children are " << open_children << dendl;
406   for (set<SnapRealm*>::iterator p = open_children.begin();
407        p != open_children.end(); ) {
408     SnapRealm *realm = *p;
409     if (realm != child &&
410     child->inode->is_projected_ancestor_of(realm->inode)) {
411       dout(20) << " child gets child realm " << *realm << " on " << *realm->inode << dendl;
412       realm->parent = child;
413       child->open_children.insert(realm);
414       open_children.erase(p++);
415     } else {
416       dout(20) << "    keeping child realm " << *realm << " on " << *realm->inode << dendl;
417       ++p;
418     }
419   }
420 
421   // split inodes_with_caps
422   elist<CInode*>::iterator p = inodes_with_caps.begin(member_offset(CInode, item_caps));
423   while (!p.end()) {
424     CInode *in = *p;
425     ++p;
426 
427     // does inode fall within the child realm?
428     bool under_child = false;
429 
430     if (in == child->inode) {
431       under_child = true;
432     } else {
433       CInode *t = in;
434       while (t->get_parent_dn()) {
435     t = t->get_parent_dn()->get_dir()->get_inode();
436     if (t == child->inode) {
437       under_child = true;
438       break;
439     }
440     if (t == in)
441       break;
442       }
443     }
444     if (under_child) {
445       dout(20) << " child gets " << *in << dendl;
446       in->move_to_realm(child);         //脱离containing_realm，加入到child realm
447     } else {
448       dout(20) << "    keeping " << *in << dendl;
449     }
450   }
451 
452 }
453 
454 const bufferlist& SnapRealm::get_snap_trace()
455 {
456   check_cache();
457   return cached_snap_trace;
458 }
459 
460 void SnapRealm::build_snap_trace(bufferlist& snapbl)    //将snaps和所有parents存储在snapbl中
461 {
462   SnapRealmInfo info(inode->ino(), srnode.created, srnode.seq, srnode.current_parent_since);
463 
464   if (parent) {
465     info.h.parent = parent->inode->ino();
466     if (!srnode.past_parents.empty()) {
467       snapid_t last = srnode.past_parents.rbegin()->first;
468       set<snapid_t> past;
469       snapid_t max_seq, max_last_created, max_last_destroyed;
470       build_snap_set(past, max_seq, max_last_created, max_last_destroyed, 0, last);
471       info.prior_parent_snaps.reserve(past.size());
472       for (set<snapid_t>::reverse_iterator p = past.rbegin(); p != past.rend(); ++p)
473     info.prior_parent_snaps.push_back(*p);
474       dout(10) << "build_snap_trace prior_parent_snaps from [1," << last << "] "
475            << info.prior_parent_snaps << dendl;
476     }
477   } else 
478     info.h.parent = 0;
479 
480   info.my_snaps.reserve(srnode.snaps.size());
481   for (map<snapid_t,SnapInfo>::reverse_iterator p = srnode.snaps.rbegin();
482        p != srnode.snaps.rend();
483        ++p)
484     info.my_snaps.push_back(p->first);
485   dout(10) << "build_snap_trace my_snaps " << info.my_snaps << dendl;
486 
487   ::encode(info, snapbl);
488 
489   if (parent)
490     parent->build_snap_trace(snapbl);
491 }
492 
493 
494 //删除past_parents中有但cached_snaps中没有的
495 void SnapRealm::prune_past_parents()
496 {
497   dout(10) << "prune_past_parents" << dendl;
498   check_cache();
499   assert(open);
500 
501   map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.begin();
502   while (p != srnode.past_parents.end()) {
503     set<snapid_t>::iterator q = cached_snaps.lower_bound(p->second.first);
504     if (q == cached_snaps.end() ||   //若cached_snaps中没有p指向的值则erase
505     *q > p->first) {
506       dout(10) << "prune_past_parents pruning [" << p->second.first << "," << p->first 
507            << "] " << p->second.ino << dendl;
508       srnode.past_parents.erase(p++);
509     } else {
510       dout(10) << "prune_past_parents keeping [" << p->second.first << "," << p->first 
511            << "] " << p->second.ino << dendl;
512       ++p;
513     }
514   }
515 }

  1 #include "SnapServer.h"
  2 #include "MDS.h"
  3 #include "osd/OSDMap.h"
  4 #include "osdc/Objecter.h"
  5 #include "mon/MonClient.h"
  6 
  7 #include "include/types.h"
  8 #include "messages/MMDSTableRequest.h"
  9 #include "messages/MRemoveSnaps.h"
 10 
 11 #include "msg/Messenger.h"
 12 
 13 #include "common/config.h"
 14 #include "include/assert.h"
 15 
 16 #define dout_subsys ceph_subsys_mds
 17 #undef dout_prefix
 18 #define dout_prefix *_dout << "mds." << rank << ".snap "
 19 
 20 
 21 void SnapServer::reset_state()  //重置状态,将snaps和need_to_purge中的内容清空
 22 {
 23   last_snap = 1;  /* snapid 1 reserved for initial root snaprealm */
 24   snaps.clear();
 25   need_to_purge.clear();
 26 
 27   // find any removed snapshot in data pools
 28   snapid_t first_free = 0;
 29   const OSDMap *osdmap = mds->objecter->get_osdmap_read();
 30   for (set<int64_t>::const_iterator p = mds->mdsmap->get_data_pools().begin();
 31        p != mds->mdsmap->get_data_pools().end();
 32        ++p) {
 33     const pg_pool_t *pi = osdmap->get_pg_pool(*p);
 34     if (!pi->removed_snaps.empty() &&
 35         pi->removed_snaps.range_end() > first_free)
 36       first_free = pi->removed_snaps.range_end();
 37   }
 38   mds->objecter->put_osdmap_read();
 39   if (first_free > last_snap)
 40     last_snap = first_free;        //扩展last_snap范围
 41 }
 42 
 43 
 44 // SERVER
 45 //根据操作执行:创建-将bl中的内容加入到pending_creat中，销毁-将bl中的内容加入到pending_destroy中
 46 void SnapServer::_prepare(bufferlist &bl, uint64_t reqid, mds_rank_t bymds)
 47 {
 48   bufferlist::iterator p = bl.begin();
 49   __u32 op;
 50   ::decode(op, p);
 51 
 52   switch (op) {
 53   case TABLE_OP_CREATE:
 54     {
 55       version++;
 56 
 57       SnapInfo info;
 58       ::decode(info.ino, p);
 59       if (!p.end()) {
 60     ::decode(info.name, p);
 61     ::decode(info.stamp, p);
 62     info.snapid = ++last_snap;
 63     pending_create[version] = info;
 64     dout(10) << "prepare v" << version << " create " << info << dendl;
 65       } else {
 66     pending_noop.insert(version);
 67     dout(10) << "prepare v" << version << " noop" << dendl;
 68       }
 69       bl.clear();
 70       ::encode(last_snap, bl);
 71     }
 72     break;
 73 
 74   case TABLE_OP_DESTROY:
 75     {
 76       inodeno_t ino;
 77       snapid_t snapid;
 78       ::decode(ino, p);    // not used, currently.
 79       ::decode(snapid, p);
 80       version++;
 81 
 82       // bump last_snap... we use it as a version value on the snaprealm.
 83       ++last_snap;
 84 
 85       pending_destroy[version] = pair<snapid_t,snapid_t>(snapid, last_snap);
 86       dout(10) << "prepare v" << version << " destroy " << snapid << " seq " << last_snap << dendl;
 87 
 88       bl.clear();
 89       ::encode(last_snap, bl);
 90     }
 91     break;
 92 
 93   default:
 94     assert(0);
 95   }
 96   //dump();
 97 }
 98 
 99 //返回容器中此id出现的次数
100 bool SnapServer::_is_prepared(version_t tid)
101 {
102   return 
103     pending_create.count(tid) ||
104     pending_destroy.count(tid);
105 }
106 
107 //将操作pending_create或pending_destroy的内容存入对应的结构中
108 bool SnapServer::_commit(version_t tid, MMDSTableRequest *req)
109 {
110   if (pending_create.count(tid)) {    //若此tid在pending_creat中有值，清空并将此结构中对应snapid中的内容存入snaps中
111     dout(7) << "commit " << tid << " create " << pending_create[tid] << dendl;
112     snaps[pending_create[tid].snapid] = pending_create[tid];
113     pending_create.erase(tid);
114   } 
115 
116   else if (pending_destroy.count(tid)) {      //若此tid在pending_creat中有值，清空并将对应内容加入到need_to_purge
117     snapid_t sn = pending_destroy[tid].first;    //removed_snap
118     snapid_t seq = pending_destroy[tid].second;   //seq
119     dout(7) << "commit " << tid << " destroy " << sn << " seq " << seq << dendl;
120     snaps.erase(sn);
121 
122     for (set<int64_t>::const_iterator p = mds->mdsmap->get_data_pools().begin();
123      p != mds->mdsmap->get_data_pools().end();
124      ++p) {
125       need_to_purge[*p].insert(sn);
126       need_to_purge[*p].insert(seq);
127     }
128 
129     pending_destroy.erase(tid);
130   }
131   else if (pending_noop.count(tid)) {     //清空pending_noop
132     dout(7) << "commit " << tid << " noop" << dendl;
133     pending_noop.erase(tid);
134   }
135   else
136     assert(0);
137 
138   // bump version.
139   version++;
140   //dump();
141   return true;
142 }
143 
144 //输出要执行的回滚命令并清空pending_create等结构中的内容，不执行具体操作
145 void SnapServer::_rollback(version_t tid) 
146 {
147   if (pending_create.count(tid)) {
148     dout(7) << "rollback " << tid << " create " << pending_create[tid] << dendl;
149     pending_create.erase(tid);
150   } 
151 
152   else if (pending_destroy.count(tid)) {
153     dout(7) << "rollback " << tid << " destroy " << pending_destroy[tid] << dendl;
154     pending_destroy.erase(tid);
155   }
156   
157   else if (pending_noop.count(tid)) {
158     dout(7) << "rollback " << tid << " noop" << dendl;
159     pending_noop.erase(tid);
160   }    
161 
162   else
163     assert(0);
164 
165   // bump version.
166   version++;
167   //dump();
168 }
169 
170 //将传入参数bl中的项在need_to_purge的对应项擦除
171 void SnapServer::_server_update(bufferlist& bl)
172 {
173   bufferlist::iterator p = bl.begin();
174   map<int, vector<snapid_t> > purge;
175   ::decode(purge, p);
176 
177   dout(7) << "_server_update purged " << purge << dendl;
178   for (map<int, vector<snapid_t> >::iterator p = purge.begin();
179        p != purge.end();
180        ++p) {
181     for (vector<snapid_t>::iterator q = p->second.begin();
182      q != p->second.end();
183      ++q)
184       need_to_purge[p->first].erase(*q);
185     if (need_to_purge[p->first].empty())
186       need_to_purge.erase(p->first);
187   }
188 
189   version++;
190 }
191 
192 void SnapServer::handle_query(MMDSTableRequest *req)
193 {
194   req->put();
195 }
196 
197 
198 //遍历need_to_purge，执行remove的操作
199 void SnapServer::check_osd_map(bool force)
200 {
201   if (!force && version == last_checked_osdmap) {
202     dout(10) << "check_osd_map - version unchanged" << dendl;
203     return;
204   }
205   dout(10) << "check_osd_map need_to_purge=" << need_to_purge << dendl;
206 
207   map<int, vector<snapid_t> > all_purge;
208   map<int, vector<snapid_t> > all_purged;
209 
210   const OSDMap *osdmap = mds->objecter->get_osdmap_read();
211   for (map<int, set<snapid_t> >::iterator p = need_to_purge.begin();       //遍历need_to_purge
212        p != need_to_purge.end();
213        ++p) {
214     int id = p->first;
215     const pg_pool_t *pi = osdmap->get_pg_pool(id);
216     for (set<snapid_t>::iterator q = p->second.begin();
217      q != p->second.end();
218      ++q) {
219       if (pi->is_removed_snap(*q)) {     //已经remove，存入all_purged，还没有remove则存入all_purge
220     dout(10) << " osdmap marks " << *q << " as removed" << dendl;
221     all_purged[id].push_back(*q);
222       } else {
223     all_purge[id].push_back(*q);
224       }
225     }
226   }
227   mds->objecter->put_osdmap_read();
228 
229   if (!all_purged.empty()) {  //已经purge，调用do_server_update更新mdlog
230     // prepare to remove from need_to_purge list
231     bufferlist bl;
232     ::encode(all_purged, bl);
233     do_server_update(bl);
234   }
235 
236   if (!all_purge.empty()) {        //需要purge，创建消息存储all_purge并发送给mon
237     dout(10) << "requesting removal of " << all_purge << dendl;
238     MRemoveSnaps *m = new MRemoveSnaps(all_purge);
239     mds->monc->send_mon_message(m);
240   }
241 
242   last_checked_osdmap = version;
243 }
244 
245 
246 
247 void SnapServer::dump(Formatter *f) const
248 {
249   f->open_object_section("snapserver");
250 
251   f->dump_int("last_snap", last_snap.val);
252 
253   f->open_array_section("pending_noop");
254   for(set<version_t>::const_iterator i = pending_noop.begin(); i != pending_noop.end(); ++i) {
255     f->dump_unsigned("version", *i);
256   }
257   f->close_section();
258 
259   f->open_array_section("snaps");
260   for (map<snapid_t, SnapInfo>::const_iterator i = snaps.begin(); i != snaps.end(); ++i) {
261     f->open_object_section("snap");
262     i->second.dump(f);
263     f->close_section();
264   }
265   f->close_section();
266 
267   f->open_object_section("need_to_purge");
268   for (map<int, set<snapid_t> >::const_iterator i = need_to_purge.begin(); i != need_to_purge.end(); ++i) {
269     stringstream pool_id;
270     pool_id << i->first;
271     f->open_array_section(pool_id.str().c_str());
272     for (set<snapid_t>::const_iterator s = i->second.begin(); s != i->second.end(); ++s) {
273       f->dump_unsigned("snapid", s->val);
274     }
275     f->close_section();
276   }
277   f->close_section();
278 
279   f->open_array_section("pending_create");
280   for(map<version_t, SnapInfo>::const_iterator i = pending_create.begin(); i != pending_create.end(); ++i) {
281     f->open_object_section("snap");
282     f->dump_unsigned("version", i->first);
283     f->open_object_section("snapinfo");
284     i->second.dump(f);
285     f->close_section();
286     f->close_section();
287   }
288   f->close_section();
289 
290   f->open_array_section("pending_destroy");
291   for(map<version_t, pair<snapid_t, snapid_t> >::const_iterator i = pending_destroy.begin(); i != pending_destroy.end(); ++i) {
292     f->open_object_section("snap");
293     f->dump_unsigned("version", i->first);
294     f->dump_unsigned("removed_snap", i->second.first);
295     f->dump_unsigned("seq", i->second.second);
296     f->close_section();
297   }
298   f->close_section();
299 
300   f->close_section();
301 }
302 
303 void SnapServer::generate_test_instances(list<SnapServer*>& ls)
304 {
305   list<SnapInfo*> snapinfo_instances;
306   SnapInfo::generate_test_instances(snapinfo_instances);
307   SnapInfo populated_snapinfo = *(snapinfo_instances.back());
308   for (list<SnapInfo*>::iterator i = snapinfo_instances.begin(); i != snapinfo_instances.end(); ++i) {
309     delete *i;
310   }
311 
312   SnapServer *blank = new SnapServer();
313   ls.push_back(blank);
314   SnapServer *populated = new SnapServer();
315   populated->last_snap = 123;
316   populated->snaps[456] = populated_snapinfo;
317   populated->need_to_purge[2].insert(012);
318   populated->pending_create[234] = populated_snapinfo;
319   populated->pending_destroy[345].first = 567;
320   populated->pending_destroy[345].second = 768;
321   populated->pending_noop.insert(890);
322 
323   ls.push_back(populated);
324 
325 }

  1 #include "SnapMapper.h"
  2 
  3 #define dout_subsys ceph_subsys_osd
  4 #undef dout_prefix
  5 #define dout_prefix *_dout << "snap_mapper."
  6 
  7 using std::string;
  8 
  9 const string SnapMapper::MAPPING_PREFIX = "MAP_";
 10 const string SnapMapper::OBJECT_PREFIX = "OBJ_";
 11 
 12 int OSDriver::get_keys(
 13   const std::set<std::string> &keys,
 14   std::map<std::string, bufferlist> *out)
 15 {
 16   return os->omap_get_values(cid, hoid, keys, out);
 17 }
 18 
 19 int OSDriver::get_next(
 20   const std::string &key,
 21   pair<std::string, bufferlist> *next)
 22 {
 23   ObjectMap::ObjectMapIterator iter =
 24     os->get_omap_iterator(cid, hoid);
 25   if (!iter) {
 26     assert(0);
 27     return -EINVAL;
 28   }
 29   iter->upper_bound(key);
 30   if (iter->valid()) {
 31     if (next)
 32       *next = make_pair(iter->key(), iter->value());
 33     return 0;
 34   } else {
 35     return -ENOENT;
 36   }
 37 }
 38 
 39 struct Mapping {
 40   snapid_t snap;
 41   hobject_t hoid;
 42   Mapping(const pair<snapid_t, hobject_t> &in)
 43     : snap(in.first), hoid(in.second) {}
 44   Mapping() : snap(0) {}
 45   void encode(bufferlist &bl) const {
 46     ENCODE_START(1, 1, bl);
 47     ::encode(snap, bl);
 48     ::encode(hoid, bl);
 49     ENCODE_FINISH(bl);
 50   }
 51   void decode(bufferlist::iterator &bl) {
 52     DECODE_START(1, bl);
 53     ::decode(snap, bl);
 54     ::decode(hoid, bl);
 55     DECODE_FINISH(bl);
 56   }
 57 };
 58 WRITE_CLASS_ENCODER(Mapping)
 59 
 60 string SnapMapper::get_prefix(snapid_t snap)
 61 {
 62   char buf[100];
 63   int len = snprintf(
 64     buf, sizeof(buf),
 65     "%.*X_", (int)(sizeof(snap)*2),
 66     static_cast<unsigned>(snap));
 67   return MAPPING_PREFIX + string(buf, len);
 68 }
 69 
 70 string SnapMapper::to_raw_key(            //实现mapping有两种key，此为raw_key
 71   const pair<snapid_t, hobject_t> &in)
 72 {
 73   return get_prefix(in.first) + shard_prefix + in.second.to_str();
 74 }
 75 
 76 pair<string, bufferlist> SnapMapper::to_raw(    //返回键值和对应map
 77   const pair<snapid_t, hobject_t> &in)
 78 {
 79   bufferlist bl;
 80   ::encode(Mapping(in), bl);
 81   return make_pair(
 82     to_raw_key(in),
 83     bl);
 84 }
 85 
 86 pair<snapid_t, hobject_t> SnapMapper::from_raw(    //根据map返回snapid和hobject对应的snap和hoid 与to_raw相反
 87   const pair<std::string, bufferlist> &image)
 88 {
 89   Mapping map;
 90   bufferlist bl(image.second);
 91   bufferlist::iterator bp(bl.begin());
 92   ::decode(map, bp);
 93   return make_pair(map.snap, map.hoid);
 94 }
 95 
 96 bool SnapMapper::is_mapping(const string &to_test)
 97 {
 98   return to_test.substr(0, MAPPING_PREFIX.size()) == MAPPING_PREFIX;
 99 }
100 
101 string SnapMapper::to_object_key(const hobject_t &hoid)          //实现mapping有两种key，此为object_key
102 {
103   return OBJECT_PREFIX + shard_prefix + hoid.to_str();
104 }
105 
106 void SnapMapper::object_snaps::encode(bufferlist &bl) const
107 {
108   ENCODE_START(1, 1, bl);
109   ::encode(oid, bl);
110   ::encode(snaps, bl);
111   ENCODE_FINISH(bl);
112 }
113 
114 void SnapMapper::object_snaps::decode(bufferlist::iterator &bl)
115 {
116   DECODE_START(1, bl);
117   ::decode(oid, bl);
118   ::decode(snaps, bl);
119   DECODE_FINISH(bl);
120 }
121 
122 int SnapMapper::get_snaps(      //根据iod从backend中找到对应的snaps并输出
123   const hobject_t &oid,
124   object_snaps *out)
125 {
126   assert(check(oid));
127   set<string> keys;
128   map<string, bufferlist> got;
129   keys.insert(to_object_key(oid));
130   int r = backend.get_keys(keys, &got);
131   if (r < 0)
132     return r;
133   if (got.empty())
134     return -ENOENT;
135   if (out) {
136     bufferlist::iterator bp = got.begin()->second.begin();
137     ::decode(*out, bp);
138     dout(20) << __func__ << " " << oid << " " << out->snaps << dendl;
139     assert(!out->snaps.empty());
140   } else {
141     dout(20) << __func__ << " " << oid << " (out == NULL)" << dendl;
142   }
143   return 0;
144 }
145 
146 void SnapMapper::clear_snaps(         //remove to_object_key
147   const hobject_t &oid,
148   MapCacher::Transaction<std::string, bufferlist> *t)
149 {
150   assert(check(oid));
151   set<string> to_remove;
152   to_remove.insert(to_object_key(oid));
153   backend.remove_keys(to_remove, t);
154 }
155 
156 void SnapMapper::set_snaps(           //将object_key和snap设置为一个map，并调用set_keys加入到Transaction结构中
157   const hobject_t &oid,
158   const object_snaps &in,
159   MapCacher::Transaction<std::string, bufferlist> *t)
160 {
161   assert(check(oid));
162   map<string, bufferlist> to_set;
163   bufferlist bl;
164   ::encode(in, bl);
165   to_set[to_object_key(oid)] = bl;
166   backend.set_keys(to_set, t);
167 }
168 
169 int SnapMapper::update_snaps(          //将new_snaps更新加入backend中
170   const hobject_t &oid,
171   const set<snapid_t> &new_snaps,
172   const set<snapid_t> *old_snaps_check,
173   MapCacher::Transaction<std::string, bufferlist> *t)
174 {
175   dout(20) << __func__ << " " << oid << " " << new_snaps
176        << " was " << (old_snaps_check ? *old_snaps_check : set<snapid_t>())
177        << dendl;
178   assert(check(oid));
179   if (new_snaps.empty())
180     return remove_oid(oid, t);
181 
182   object_snaps out;
183   int r = get_snaps(oid, &out);
184   if (r < 0)
185     return r;
186   if (old_snaps_check)
187     assert(out.snaps == *old_snaps_check);
188 
189   object_snaps in(oid, new_snaps);
190   set_snaps(oid, in, t);
191 
192   set<string> to_remove;
193   for (set<snapid_t>::iterator i = out.snaps.begin();
194        i != out.snaps.end();
195        ++i) {
196     if (!new_snaps.count(*i)) {
197       to_remove.insert(to_raw_key(make_pair(*i, oid)));
198     }
199   }
200   backend.remove_keys(to_remove, t);
201   return 0;
202 }
203 
204 void SnapMapper::add_oid(
205   const hobject_t &oid,
206   const set<snapid_t>& snaps,
207   MapCacher::Transaction<std::string, bufferlist> *t)
208 {
209   dout(20) << __func__ << " " << oid << " " << snaps << dendl;
210   assert(check(oid));
211   {
212     object_snaps out;
213     int r = get_snaps(oid, &out);
214     assert(r == -ENOENT);
215   }
216 
217   object_snaps _snaps(oid, snaps);
218   set_snaps(oid, _snaps, t);
219 
220   map<string, bufferlist> to_add;
221   for (set<snapid_t>::iterator i = snaps.begin();
222        i != snaps.end();
223        ++i) {
224     to_add.insert(to_raw(make_pair(*i, oid)));
225   }
226   backend.set_keys(to_add, t);  //将to_add和t打包加入backend
227 }
228 
229 int SnapMapper::get_next_object_to_trim(
230   snapid_t snap,
231   hobject_t *hoid)
232 {
233   for (set<string>::iterator i = prefixes.begin();
234        i != prefixes.end();
235        ++i) {
236     string list_after(get_prefix(snap) + *i);
237 
238     pair<string, bufferlist> next;
239     int r = backend.get_next(list_after, &next);
240     if (r < 0) {
241       break; // Done
242     }
243 
244     if (next.first.substr(0, list_after.size()) !=
245     list_after) {
246       continue; // Done with this prefix
247     }
248 
249     assert(is_mapping(next.first));
250 
251     pair<snapid_t, hobject_t> next_decoded(from_raw(next));
252     assert(next_decoded.first == snap);
253     assert(check(next_decoded.second));
254 
255     if (hoid)
256       *hoid = next_decoded.second;
257     return 0;
258   }
259   return -ENOENT;
260 }
261 
262 
263 int SnapMapper::remove_oid(
264   const hobject_t &oid,
265   MapCacher::Transaction<std::string, bufferlist> *t)
266 {
267   dout(20) << __func__ << " " << oid << dendl;
268   assert(check(oid));
269   return _remove_oid(oid, t);
270 }
271 
272 int SnapMapper::_remove_oid(
273   const hobject_t &oid,
274   MapCacher::Transaction<std::string, bufferlist> *t)
275 {
276   object_snaps out;
277   int r = get_snaps(oid, &out);
278   if (r < 0)
279     return r;
280 
281   clear_snaps(oid, t);       //remove to_object_key
282 
283   set<string> to_remove;
284   for (set<snapid_t>::iterator i = out.snaps.begin();
285        i != out.snaps.end();
286        ++i) {
287     to_remove.insert(to_raw_key(make_pair(*i, oid)));
288   }
289   backend.remove_keys(to_remove, t);    //remove to_raw_key
290   return 0;
291 }
292 
293 int SnapMapper::get_snaps(         //根据oid找到对应snaps并存在set容器snaps中
294   const hobject_t &oid,
295   std::set<snapid_t> *snaps)
296 {
297   assert(check(oid));
298   object_snaps out;
299   int r = get_snaps(oid, &out);
300   if (r < 0)
301     return r;
302   if (snaps)
303     snaps->swap(out.snaps);
304   return 0;
305 }