1#!/usr/bin/python3 

2# 

3# Copyright (C) Citrix Systems Inc. 

4# 

5# This program is free software; you can redistribute it and/or modify 

6# it under the terms of the GNU Lesser General Public License as published 

7# by the Free Software Foundation; version 2.1 only. 

8# 

9# This program is distributed in the hope that it will be useful, 

10# but WITHOUT ANY WARRANTY; without even the implied warranty of 

11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

12# GNU Lesser General Public License for more details. 

13# 

14# You should have received a copy of the GNU Lesser General Public License 

15# along with this program; if not, write to the Free Software Foundation, Inc., 

16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 

17# 

18# Script to coalesce and garbage collect COW-based SR's in the background 

19# 

20 

21from sm_typing import Any, Dict, Optional, List, override 

22 

23import os 

24import os.path 

25import sys 

26import time 

27import signal 

28import subprocess 

29import getopt 

30import datetime 

31import traceback 

32import base64 

33import zlib 

34import errno 

35import stat 

36 

37import XenAPI # pylint: disable=import-error 

38import util 

39import lvutil 

40import lvmcache 

41import journaler 

42import fjournaler 

43import lock 

44import blktap2 

45import xs_errors 

46from refcounter import RefCounter 

47from ipc import IPCFlag 

48from lvmanager import LVActivator 

49from srmetadata import LVMMetadataHandler, VDI_TYPE_TAG 

50from functools import reduce 

51from time import monotonic as _time 

52 

53from constants import NS_PREFIX_LVM, VG_LOCATION, VG_PREFIX 

54from cowutil import CowImageInfo, CowUtil, getCowUtil 

55from lvmcowutil import LV_PREFIX, LvmCowUtil 

56from vditype import VdiType, VdiTypeExtension, VDI_COW_TYPES, VDI_TYPE_TO_EXTENSION 

57 

58try: 

59 from linstorcowutil import LinstorCowUtil, MultiLinstorCowUtil 

60 from linstorjournaler import LinstorJournaler 

61 from linstorvolumemanager import get_controller_uri 

62 from linstorvolumemanager import LinstorVolumeManager, LinstorVolumeManagerError, LinstorVolumeOpeners 

63 from linstorvolumemanager import PERSISTENT_PREFIX as LINSTOR_PERSISTENT_PREFIX 

64 

65 LINSTOR_AVAILABLE = True 

66except ImportError: 

67 LINSTOR_AVAILABLE = False 

68 

69# Disable automatic leaf-coalescing. Online leaf-coalesce is currently not 

70# possible due to lvhd_stop_using_() not working correctly. However, we leave 

71# this option available through the explicit LEAFCLSC_FORCE flag in the VDI 

72# record for use by the offline tool (which makes the operation safe by pausing 

73# the VM first) 

74AUTO_ONLINE_LEAF_COALESCE_ENABLED = True 

75 

76FLAG_TYPE_ABORT = "abort" # flag to request aborting of GC/coalesce 

77 

78# process "lock", used simply as an indicator that a process already exists 

79# that is doing GC/coalesce on this SR (such a process holds the lock, and we 

80# check for the fact by trying the lock). 

81lockGCRunning = None 

82 

83# process "lock" to indicate that the GC process has been activated but may not 

84# yet be running, stops a second process from being started. 

85LOCK_TYPE_GC_ACTIVE = "gc_active" 

86lockGCActive = None 

87 

88# Default coalesce error rate limit, in messages per minute. A zero value 

89# disables throttling, and a negative value disables error reporting. 

90DEFAULT_COALESCE_ERR_RATE = 1.0 / 60 

91 

92COALESCE_LAST_ERR_TAG = 'last-coalesce-error' 

93COALESCE_ERR_RATE_TAG = 'coalesce-error-rate' 

94VAR_RUN = "/var/run/" 

95SPEED_LOG_ROOT = VAR_RUN + "{uuid}.speed_log" 

96 

97N_RUNNING_AVERAGE = 10 

98 

99NON_PERSISTENT_DIR = '/run/nonpersistent/sm' 

100 

101# Signal Handler 

102SIGTERM = False 

103 

104 

105class AbortException(util.SMException): 

106 pass 

107 

108class CancelException(util.SMException): 

109 pass 

110 

111def receiveSignal(signalNumber, frame): 

112 global SIGTERM 

113 

114 util.SMlog("GC: recieved SIGTERM") 

115 SIGTERM = True 

116 return 

117 

118 

119################################################################################ 

120# 

121# Util 

122# 

123class Util: 

124 RET_RC = 1 

125 RET_STDOUT = 2 

126 RET_STDERR = 4 

127 

128 UUID_LEN = 36 

129 

130 PREFIX = {"G": 1024 * 1024 * 1024, "M": 1024 * 1024, "K": 1024} 

131 

132 @staticmethod 

133 def log(text) -> None: 

134 util.SMlog(text, ident="SMGC") 

135 

136 @staticmethod 

137 def logException(tag): 

138 info = sys.exc_info() 

139 if info[0] == SystemExit: 139 ↛ 141line 139 didn't jump to line 141, because the condition on line 139 was never true

140 # this should not be happening when catching "Exception", but it is 

141 sys.exit(0) 

142 tb = reduce(lambda a, b: "%s%s" % (a, b), traceback.format_tb(info[2])) 

143 Util.log("*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*") 

144 Util.log(" ***********************") 

145 Util.log(" * E X C E P T I O N *") 

146 Util.log(" ***********************") 

147 Util.log("%s: EXCEPTION %s, %s" % (tag, info[0], info[1])) 

148 Util.log(tb) 

149 Util.log("*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*") 

150 

151 @staticmethod 

152 def doexec(args, expectedRC, inputtext=None, ret=None, log=True): 

153 "Execute a subprocess, then return its return code, stdout, stderr" 

154 proc = subprocess.Popen(args, 

155 stdin=subprocess.PIPE, \ 

156 stdout=subprocess.PIPE, \ 

157 stderr=subprocess.PIPE, \ 

158 shell=True, \ 

159 close_fds=True) 

160 (stdout, stderr) = proc.communicate(inputtext) 

161 stdout = str(stdout) 

162 stderr = str(stderr) 

163 rc = proc.returncode 

164 if log: 

165 Util.log("`%s`: %s" % (args, rc)) 

166 if type(expectedRC) != type([]): 

167 expectedRC = [expectedRC] 

168 if not rc in expectedRC: 

169 reason = stderr.strip() 

170 if stdout.strip(): 

171 reason = "%s (stdout: %s)" % (reason, stdout.strip()) 

172 Util.log("Failed: %s" % reason) 

173 raise util.CommandException(rc, args, reason) 

174 

175 if ret == Util.RET_RC: 

176 return rc 

177 if ret == Util.RET_STDERR: 

178 return stderr 

179 return stdout 

180 

181 @staticmethod 

182 def runAbortable(func, ret, ns, abortTest, pollInterval, timeOut, prefSig=signal.SIGKILL): 

183 """execute func in a separate thread and kill it if abortTest signals 

184 so""" 

185 abortSignaled = abortTest() # check now before we clear resultFlag 

186 resultFlag = IPCFlag(ns) 

187 resultFlag.clearAll() 

188 pid = os.fork() 

189 if pid: 

190 startTime = _time() 

191 try: 

192 while True: 

193 if resultFlag.test("success"): 

194 Util.log(" Child process completed successfully") 

195 resultFlag.clear("success") 

196 return 

197 if resultFlag.test("failure"): 

198 resultFlag.clear("failure") 

199 raise util.SMException("Child process exited with error") 

200 if abortTest() or abortSignaled or SIGTERM: 

201 os.killpg(pid, prefSig) 

202 raise AbortException("Aborting due to signal") 

203 if timeOut and _time() - startTime > timeOut: 

204 os.killpg(pid, prefSig) 

205 resultFlag.clearAll() 

206 raise util.SMException("Timed out") 

207 time.sleep(pollInterval) 

208 finally: 

209 wait_pid = 0 

210 rc = -1 

211 count = 0 

212 while wait_pid == 0 and count < 10: 

213 wait_pid, rc = os.waitpid(pid, os.WNOHANG) 

214 if wait_pid == 0: 

215 time.sleep(2) 

216 count += 1 

217 

218 if wait_pid == 0: 

219 Util.log("runAbortable: wait for process completion timed out") 

220 else: 

221 os.setpgrp() 

222 try: 

223 if func() == ret: 

224 resultFlag.set("success") 

225 else: 

226 resultFlag.set("failure") 

227 except Exception as e: 

228 Util.log("Child process failed with : (%s)" % e) 

229 resultFlag.set("failure") 

230 Util.logException("This exception has occured") 

231 os._exit(0) 

232 

233 @staticmethod 

234 def num2str(number): 

235 for prefix in ("G", "M", "K"): 235 ↛ 238line 235 didn't jump to line 238, because the loop on line 235 didn't complete

236 if number >= Util.PREFIX[prefix]: 

237 return "%.3f%s" % (float(number) / Util.PREFIX[prefix], prefix) 

238 return "%s" % number 

239 

240 @staticmethod 

241 def numBits(val): 

242 count = 0 

243 while val: 

244 count += val & 1 

245 val = val >> 1 

246 return count 

247 

248 @staticmethod 

249 def countBits(bitmap1, bitmap2): 

250 """return bit count in the bitmap produced by ORing the two bitmaps""" 

251 len1 = len(bitmap1) 

252 len2 = len(bitmap2) 

253 lenLong = len1 

254 lenShort = len2 

255 bitmapLong = bitmap1 

256 if len2 > len1: 

257 lenLong = len2 

258 lenShort = len1 

259 bitmapLong = bitmap2 

260 

261 count = 0 

262 for i in range(lenShort): 

263 val = bitmap1[i] | bitmap2[i] 

264 count += Util.numBits(val) 

265 

266 for i in range(i + 1, lenLong): 

267 val = bitmapLong[i] 

268 count += Util.numBits(val) 

269 return count 

270 

271 @staticmethod 

272 def getThisScript(): 

273 thisScript = util.get_real_path(__file__) 

274 if thisScript.endswith(".pyc"): 

275 thisScript = thisScript[:-1] 

276 return thisScript 

277 

278 

279################################################################################ 

280# 

281# XAPI 

282# 

283class XAPI: 

284 USER = "root" 

285 PLUGIN_ON_SLAVE = "on-slave" 

286 

287 CONFIG_SM = 0 

288 CONFIG_OTHER = 1 

289 CONFIG_ON_BOOT = 2 

290 CONFIG_ALLOW_CACHING = 3 

291 

292 CONFIG_NAME = { 

293 CONFIG_SM: "sm-config", 

294 CONFIG_OTHER: "other-config", 

295 CONFIG_ON_BOOT: "on-boot", 

296 CONFIG_ALLOW_CACHING: "allow_caching" 

297 } 

298 

299 class LookupError(util.SMException): 

300 pass 

301 

302 @staticmethod 

303 def getSession(): 

304 session = XenAPI.xapi_local() 

305 session.xenapi.login_with_password(XAPI.USER, '', '', 'SM') 

306 return session 

307 

308 def __init__(self, session, srUuid): 

309 self.sessionPrivate = False 

310 self.session = session 

311 if self.session is None: 

312 self.session = self.getSession() 

313 self.sessionPrivate = True 

314 self._srRef = self.session.xenapi.SR.get_by_uuid(srUuid) 

315 self.srRecord = self.session.xenapi.SR.get_record(self._srRef) 

316 self.hostUuid = util.get_this_host() 

317 self._hostRef = self.session.xenapi.host.get_by_uuid(self.hostUuid) 

318 self.task = None 

319 self.task_progress = {"coalescable": 0, "done": 0} 

320 

321 def __del__(self): 

322 if self.sessionPrivate: 

323 self.session.xenapi.session.logout() 

324 

325 @property 

326 def srRef(self): 

327 return self._srRef 

328 

329 def isPluggedHere(self): 

330 pbds = self.getAttachedPBDs() 

331 for pbdRec in pbds: 

332 if pbdRec["host"] == self._hostRef: 

333 return True 

334 return False 

335 

336 def poolOK(self): 

337 host_recs = self.session.xenapi.host.get_all_records() 

338 for host_ref, host_rec in host_recs.items(): 

339 if not host_rec["enabled"]: 

340 Util.log("Host %s not enabled" % host_rec["uuid"]) 

341 return False 

342 return True 

343 

344 def isMaster(self): 

345 if self.srRecord["shared"]: 

346 pool = list(self.session.xenapi.pool.get_all_records().values())[0] 

347 return pool["master"] == self._hostRef 

348 else: 

349 pbds = self.getAttachedPBDs() 

350 if len(pbds) < 1: 

351 raise util.SMException("Local SR not attached") 

352 elif len(pbds) > 1: 

353 raise util.SMException("Local SR multiply attached") 

354 return pbds[0]["host"] == self._hostRef 

355 

356 def getAttachedPBDs(self): 

357 """Return PBD records for all PBDs of this SR that are currently 

358 attached""" 

359 attachedPBDs = [] 

360 pbds = self.session.xenapi.PBD.get_all_records() 

361 for pbdRec in pbds.values(): 

362 if pbdRec["SR"] == self._srRef and pbdRec["currently_attached"]: 

363 attachedPBDs.append(pbdRec) 

364 return attachedPBDs 

365 

366 def getOnlineHosts(self): 

367 return util.get_online_hosts(self.session) 

368 

369 def ensureInactive(self, hostRef, args): 

370 text = self.session.xenapi.host.call_plugin( \ 

371 hostRef, self.PLUGIN_ON_SLAVE, "multi", args) 

372 Util.log("call-plugin returned: '%s'" % text) 

373 

374 def getRecordHost(self, hostRef): 

375 return self.session.xenapi.host.get_record(hostRef) 

376 

377 def _getRefVDI(self, uuid): 

378 return self.session.xenapi.VDI.get_by_uuid(uuid) 

379 

380 def getRefVDI(self, vdi): 

381 return self._getRefVDI(vdi.uuid) 

382 

383 def getRecordVDI(self, uuid): 

384 try: 

385 ref = self._getRefVDI(uuid) 

386 return self.session.xenapi.VDI.get_record(ref) 

387 except XenAPI.Failure: 

388 return None 

389 

390 def singleSnapshotVDI(self, vdi): 

391 return self.session.xenapi.VDI.snapshot(vdi.getRef(), 

392 {"type": "internal"}) 

393 

394 def forgetVDI(self, srUuid, vdiUuid): 

395 """Forget the VDI, but handle the case where the VDI has already been 

396 forgotten (i.e. ignore errors)""" 

397 try: 

398 vdiRef = self.session.xenapi.VDI.get_by_uuid(vdiUuid) 

399 self.session.xenapi.VDI.forget(vdiRef) 

400 except XenAPI.Failure: 

401 pass 

402 

403 def getConfigVDI(self, vdi, key): 

404 kind = vdi.CONFIG_TYPE[key] 

405 if kind == self.CONFIG_SM: 

406 cfg = self.session.xenapi.VDI.get_sm_config(vdi.getRef()) 

407 elif kind == self.CONFIG_OTHER: 

408 cfg = self.session.xenapi.VDI.get_other_config(vdi.getRef()) 

409 elif kind == self.CONFIG_ON_BOOT: 

410 cfg = self.session.xenapi.VDI.get_on_boot(vdi.getRef()) 

411 elif kind == self.CONFIG_ALLOW_CACHING: 

412 cfg = self.session.xenapi.VDI.get_allow_caching(vdi.getRef()) 

413 else: 

414 assert(False) 

415 Util.log("Got %s for %s: %s" % (self.CONFIG_NAME[kind], vdi, repr(cfg))) 

416 return cfg 

417 

418 def removeFromConfigVDI(self, vdi, key): 

419 kind = vdi.CONFIG_TYPE[key] 

420 if kind == self.CONFIG_SM: 

421 self.session.xenapi.VDI.remove_from_sm_config(vdi.getRef(), key) 

422 elif kind == self.CONFIG_OTHER: 

423 self.session.xenapi.VDI.remove_from_other_config(vdi.getRef(), key) 

424 else: 

425 assert(False) 

426 

427 def addToConfigVDI(self, vdi, key, val): 

428 kind = vdi.CONFIG_TYPE[key] 

429 if kind == self.CONFIG_SM: 

430 self.session.xenapi.VDI.add_to_sm_config(vdi.getRef(), key, val) 

431 elif kind == self.CONFIG_OTHER: 

432 self.session.xenapi.VDI.add_to_other_config(vdi.getRef(), key, val) 

433 else: 

434 assert(False) 

435 

436 def isSnapshot(self, vdi): 

437 return self.session.xenapi.VDI.get_is_a_snapshot(vdi.getRef()) 

438 

439 def markCacheSRsDirty(self): 

440 sr_refs = self.session.xenapi.SR.get_all_records_where( \ 

441 'field "local_cache_enabled" = "true"') 

442 for sr_ref in sr_refs: 

443 Util.log("Marking SR %s dirty" % sr_ref) 

444 util.set_dirty(self.session, sr_ref) 

445 

446 def srUpdate(self): 

447 Util.log("Starting asynch srUpdate for SR %s" % self.srRecord["uuid"]) 

448 abortFlag = IPCFlag(self.srRecord["uuid"]) 

449 task = self.session.xenapi.Async.SR.update(self._srRef) 

450 cancelTask = True 

451 try: 

452 for i in range(60): 

453 status = self.session.xenapi.task.get_status(task) 

454 if not status == "pending": 

455 Util.log("SR.update_asynch status changed to [%s]" % status) 

456 cancelTask = False 

457 return 

458 if abortFlag.test(FLAG_TYPE_ABORT): 

459 Util.log("Abort signalled during srUpdate, cancelling task...") 

460 try: 

461 self.session.xenapi.task.cancel(task) 

462 cancelTask = False 

463 Util.log("Task cancelled") 

464 except: 

465 pass 

466 return 

467 time.sleep(1) 

468 finally: 

469 if cancelTask: 

470 self.session.xenapi.task.cancel(task) 

471 self.session.xenapi.task.destroy(task) 

472 Util.log("Asynch srUpdate still running, but timeout exceeded.") 

473 

474 def update_task(self): 

475 self.session.xenapi.task.set_other_config( 

476 self.task, 

477 { 

478 "applies_to": self._srRef 

479 }) 

480 total = self.task_progress['coalescable'] + self.task_progress['done'] 

481 if (total > 0): 

482 self.session.xenapi.task.set_progress( 

483 self.task, float(self.task_progress['done']) / total) 

484 

485 def create_task(self, label, description): 

486 self.task = self.session.xenapi.task.create(label, description) 

487 self.update_task() 

488 

489 def update_task_progress(self, key, value): 

490 self.task_progress[key] = value 

491 if self.task: 

492 self.update_task() 

493 

494 def set_task_status(self, status): 

495 if self.task: 

496 self.session.xenapi.task.set_status(self.task, status) 

497 

498 

499################################################################################ 

500# 

501# VDI 

502# 

503class VDI(object): 

504 """Object representing a VDI of a COW-based SR""" 

505 

506 POLL_INTERVAL = 1 

507 POLL_TIMEOUT = 30 

508 DEVICE_MAJOR = 202 

509 

510 # config keys & values 

511 DB_VDI_PARENT = "vhd-parent" 

512 DB_VDI_TYPE = "vdi_type" 

513 DB_VDI_BLOCKS = "vhd-blocks" 

514 DB_VDI_PAUSED = "paused" 

515 DB_VDI_RELINKING = "relinking" 

516 DB_VDI_ACTIVATING = "activating" 

517 DB_GC = "gc" 

518 DB_COALESCE = "coalesce" 

519 DB_LEAFCLSC = "leaf-coalesce" # config key 

520 DB_GC_NO_SPACE = "gc_no_space" 

521 LEAFCLSC_DISABLED = "false" # set by user; means do not leaf-coalesce 

522 LEAFCLSC_FORCE = "force" # set by user; means skip snap-coalesce 

523 LEAFCLSC_OFFLINE = "offline" # set here for informational purposes: means 

524 # no space to snap-coalesce or unable to keep 

525 # up with VDI. This is not used by the SM, it 

526 # might be used by external components. 

527 DB_ONBOOT = "on-boot" 

528 ONBOOT_RESET = "reset" 

529 DB_ALLOW_CACHING = "allow_caching" 

530 

531 CONFIG_TYPE = { 

532 DB_VDI_PARENT: XAPI.CONFIG_SM, 

533 DB_VDI_TYPE: XAPI.CONFIG_SM, 

534 DB_VDI_BLOCKS: XAPI.CONFIG_SM, 

535 DB_VDI_PAUSED: XAPI.CONFIG_SM, 

536 DB_VDI_RELINKING: XAPI.CONFIG_SM, 

537 DB_VDI_ACTIVATING: XAPI.CONFIG_SM, 

538 DB_GC: XAPI.CONFIG_OTHER, 

539 DB_COALESCE: XAPI.CONFIG_OTHER, 

540 DB_LEAFCLSC: XAPI.CONFIG_OTHER, 

541 DB_ONBOOT: XAPI.CONFIG_ON_BOOT, 

542 DB_ALLOW_CACHING: XAPI.CONFIG_ALLOW_CACHING, 

543 DB_GC_NO_SPACE: XAPI.CONFIG_SM 

544 } 

545 

546 LIVE_LEAF_COALESCE_MAX_SIZE = 20 * 1024 * 1024 # bytes 

547 LIVE_LEAF_COALESCE_TIMEOUT = 10 # seconds 

548 TIMEOUT_SAFETY_MARGIN = 0.5 # extra margin when calculating 

549 # feasibility of leaf coalesce 

550 

551 JRN_RELINK = "relink" # journal entry type for relinking children 

552 JRN_COALESCE = "coalesce" # to communicate which VDI is being coalesced 

553 JRN_LEAF = "leaf" # used in coalesce-leaf 

554 

555 STR_TREE_INDENT = 4 

556 

557 def __init__(self, sr, uuid, vdi_type): 

558 self.sr = sr 

559 self.scanError = True 

560 self.uuid = uuid 

561 self.vdi_type = vdi_type 

562 self.fileName = "" 

563 self.parentUuid = "" 

564 self.sizeVirt = -1 

565 self._sizePhys = -1 

566 self._sizeAllocated = -1 

567 self._hidden = False 

568 self.parent = None 

569 self.children = [] 

570 self._vdiRef = None 

571 self.cowutil = getCowUtil(vdi_type) 

572 self._clearRef() 

573 

574 @staticmethod 

575 def extractUuid(path): 

576 raise NotImplementedError("Implement in sub class") 

577 

578 def load(self, info=None) -> None: 

579 """Load VDI info""" 

580 pass 

581 

582 def getDriverName(self) -> str: 

583 return self.vdi_type 

584 

585 def getRef(self): 

586 if self._vdiRef is None: 

587 self._vdiRef = self.sr.xapi.getRefVDI(self) 

588 return self._vdiRef 

589 

590 def getConfig(self, key, default=None): 

591 config = self.sr.xapi.getConfigVDI(self, key) 

592 if key == self.DB_ONBOOT or key == self.DB_ALLOW_CACHING: 592 ↛ 593line 592 didn't jump to line 593, because the condition on line 592 was never true

593 val = config 

594 else: 

595 val = config.get(key) 

596 if val: 

597 return val 

598 return default 

599 

600 def setConfig(self, key, val): 

601 self.sr.xapi.removeFromConfigVDI(self, key) 

602 self.sr.xapi.addToConfigVDI(self, key, val) 

603 Util.log("Set %s = %s for %s" % (key, val, self)) 

604 

605 def delConfig(self, key): 

606 self.sr.xapi.removeFromConfigVDI(self, key) 

607 Util.log("Removed %s from %s" % (key, self)) 

608 

609 def ensureUnpaused(self): 

610 if self.getConfig(self.DB_VDI_PAUSED) == "true": 

611 Util.log("Unpausing VDI %s" % self) 

612 self.unpause() 

613 

614 def pause(self, failfast=False) -> None: 

615 if not blktap2.VDI.tap_pause(self.sr.xapi.session, self.sr.uuid, 

616 self.uuid, failfast): 

617 raise util.SMException("Failed to pause VDI %s" % self) 

618 

619 def _report_tapdisk_unpause_error(self): 

620 try: 

621 xapi = self.sr.xapi.session.xenapi 

622 sr_ref = xapi.SR.get_by_uuid(self.sr.uuid) 

623 msg_name = "failed to unpause tapdisk" 

624 msg_body = "Failed to unpause tapdisk for VDI %s, " \ 

625 "VMs using this tapdisk have lost access " \ 

626 "to the corresponding disk(s)" % self.uuid 

627 xapi.message.create(msg_name, "4", "SR", self.sr.uuid, msg_body) 

628 except Exception as e: 

629 util.SMlog("failed to generate message: %s" % e) 

630 

631 def unpause(self): 

632 if not blktap2.VDI.tap_unpause(self.sr.xapi.session, self.sr.uuid, 

633 self.uuid): 

634 self._report_tapdisk_unpause_error() 

635 raise util.SMException("Failed to unpause VDI %s" % self) 

636 

637 def refresh(self, ignoreNonexistent=True): 

638 """Pause-unpause in one step""" 

639 self.sr.lock() 

640 try: 

641 try: 

642 if not blktap2.VDI.tap_refresh(self.sr.xapi.session, 642 ↛ 644line 642 didn't jump to line 644, because the condition on line 642 was never true

643 self.sr.uuid, self.uuid): 

644 self._report_tapdisk_unpause_error() 

645 raise util.SMException("Failed to refresh %s" % self) 

646 except XenAPI.Failure as e: 

647 if util.isInvalidVDI(e) and ignoreNonexistent: 

648 Util.log("VDI %s not found, ignoring" % self) 

649 return 

650 raise 

651 finally: 

652 self.sr.unlock() 

653 

654 def isSnapshot(self): 

655 return self.sr.xapi.isSnapshot(self) 

656 

657 def isAttachedRW(self): 

658 return util.is_attached_rw( 

659 self.sr.xapi.session.xenapi.VDI.get_sm_config(self.getRef())) 

660 

661 def getVDIBlocks(self): 

662 val = self.updateBlockInfo() 

663 bitmap = zlib.decompress(base64.b64decode(val)) 

664 return bitmap 

665 

666 def isCoalesceable(self): 

667 """A VDI is coalesceable if it has no siblings and is not a leaf""" 

668 return ( 

669 not self.scanError and 

670 self.parent and 

671 len(self.parent.children) == 1 and 

672 self.isHidden() and 

673 len(self.children) > 0 and ( 

674 # Conditions below are Qcow2 specific: 

675 # A Qcow2 chain can't be coalesce with more than one leaf attached. 

676 # Put it another way, Qcow2 leaves activated on multiple hosts must 

677 # prevent the coalesce of the chain. 

678 self.vdi_type != VdiType.QCOW2 or 

679 (self.vdi_type == VdiType.QCOW2 and len(self.sr.hasLeavesAttachedOn(self)) <= 1) 

680 ) 

681 ) 

682 

683 def isLeafCoalesceable(self): 

684 """A VDI is leaf-coalesceable if it has no siblings and is a leaf""" 

685 return not self.scanError and \ 

686 self.parent and \ 

687 len(self.parent.children) == 1 and \ 

688 not self.isHidden() and \ 

689 len(self.children) == 0 

690 

691 def canLiveCoalesce(self, speed): 

692 """Can we stop-and-leaf-coalesce this VDI? The VDI must be 

693 isLeafCoalesceable() already""" 

694 feasibleSize = False 

695 allowedDownTime = \ 

696 self.TIMEOUT_SAFETY_MARGIN * self.LIVE_LEAF_COALESCE_TIMEOUT 

697 allocated_size = self.getAllocatedSize() 

698 if speed: 

699 feasibleSize = \ 

700 allocated_size // speed < allowedDownTime 

701 else: 

702 feasibleSize = \ 

703 allocated_size < self.LIVE_LEAF_COALESCE_MAX_SIZE 

704 

705 return (feasibleSize or 

706 self.getConfig(self.DB_LEAFCLSC) == self.LEAFCLSC_FORCE) 

707 

708 def getAllPrunable(self): 

709 if len(self.children) == 0: # base case 

710 # it is possible to have a hidden leaf that was recently coalesced 

711 # onto its parent, its children already relinked but not yet 

712 # reloaded - in which case it may not be garbage collected yet: 

713 # some tapdisks could still be using the file. 

714 if self.sr.journaler.get(self.JRN_RELINK, self.uuid): 

715 return [] 

716 if not self.scanError and self.isHidden(): 

717 return [self] 

718 return [] 

719 

720 thisPrunable = True 

721 vdiList = [] 

722 for child in self.children: 

723 childList = child.getAllPrunable() 

724 vdiList.extend(childList) 

725 if child not in childList: 

726 thisPrunable = False 

727 

728 # We can destroy the current VDI if all childs are hidden BUT the 

729 # current VDI must be hidden too to do that! 

730 # Example in this case (after a failed live leaf coalesce): 

731 # 

732 # SMGC: [32436] SR 07ed ('linstor-nvme-sr') (2 VDIs in 1 VHD trees): 

733 # SMGC: [32436] b5458d61(1.000G/4.127M) 

734 # SMGC: [32436] *OLD_b545(1.000G/4.129M) 

735 # 

736 # OLD_b545 is hidden and must be removed, but b5458d61 not. 

737 # Normally we are not in this function when the delete action is 

738 # executed but in `_liveLeafCoalesce`. 

739 

740 if not self.scanError and not self.isHidden() and thisPrunable: 

741 vdiList.append(self) 

742 return vdiList 

743 

744 def getSizePhys(self) -> int: 

745 return self._sizePhys 

746 

747 def getAllocatedSize(self) -> int: 

748 return self._sizeAllocated 

749 

750 def getTreeRoot(self): 

751 "Get the root of the tree that self belongs to" 

752 root = self 

753 while root.parent: 

754 root = root.parent 

755 return root 

756 

757 def getTreeHeight(self): 

758 "Get the height of the subtree rooted at self" 

759 if len(self.children) == 0: 

760 return 1 

761 

762 maxChildHeight = 0 

763 for child in self.children: 

764 childHeight = child.getTreeHeight() 

765 if childHeight > maxChildHeight: 

766 maxChildHeight = childHeight 

767 

768 return maxChildHeight + 1 

769 

770 def getAllLeaves(self) -> List["VDI"]: 

771 "Get all leaf nodes in the subtree rooted at self" 

772 if len(self.children) == 0: 

773 return [self] 

774 

775 leaves = [] 

776 for child in self.children: 

777 leaves.extend(child.getAllLeaves()) 

778 return leaves 

779 

780 def updateBlockInfo(self) -> Optional[str]: 

781 val = base64.b64encode(self._queryCowBlocks()).decode() 

782 try: 

783 self.setConfig(VDI.DB_VDI_BLOCKS, val) 

784 except Exception: 

785 if self.vdi_type != VdiType.QCOW2: 

786 raise 

787 # Sometime with QCOW2, our allocation table is too big to be stored in XAPI, in this case we do not store it 

788 # and we write `skipped` instead so that hasWork is happy (and the GC doesn't run in loop indefinitely). 

789 self.setConfig(VDI.DB_VDI_BLOCKS, "skipped") 

790 

791 return val 

792 

793 def rename(self, uuid) -> None: 

794 "Rename the VDI file" 

795 assert(not self.sr.vdis.get(uuid)) 

796 self._clearRef() 

797 oldUuid = self.uuid 

798 self.uuid = uuid 

799 self.children = [] 

800 # updating the children themselves is the responsibility of the caller 

801 del self.sr.vdis[oldUuid] 

802 self.sr.vdis[self.uuid] = self 

803 

804 def delete(self) -> None: 

805 "Physically delete the VDI" 

806 lock.Lock.cleanup(self.uuid, NS_PREFIX_LVM + self.sr.uuid) 

807 lock.Lock.cleanupAll(self.uuid) 

808 self._clear() 

809 

810 def getParent(self) -> str: 

811 return self.cowutil.getParent(self.path, lambda x: x.strip()) 811 ↛ exitline 811 didn't run the lambda on line 811

812 

813 def repair(self, parent) -> None: 

814 self.cowutil.repair(parent) 

815 

816 @override 

817 def __str__(self) -> str: 

818 strHidden = "" 

819 if self.isHidden(): 819 ↛ 820line 819 didn't jump to line 820, because the condition on line 819 was never true

820 strHidden = "*" 

821 strSizeVirt = "?" 

822 if self.sizeVirt > 0: 822 ↛ 823line 822 didn't jump to line 823, because the condition on line 822 was never true

823 strSizeVirt = Util.num2str(self.sizeVirt) 

824 strSizePhys = "?" 

825 if self._sizePhys > 0: 825 ↛ 826line 825 didn't jump to line 826, because the condition on line 825 was never true

826 strSizePhys = "/%s" % Util.num2str(self._sizePhys) 

827 strSizeAllocated = "?" 

828 if self._sizeAllocated >= 0: 

829 strSizeAllocated = "/%s" % Util.num2str(self._sizeAllocated) 

830 strType = "[{}]".format(self.vdi_type) 

831 

832 return "%s%s(%s%s%s)%s" % (strHidden, self.uuid[0:8], strSizeVirt, 

833 strSizePhys, strSizeAllocated, strType) 

834 

835 def validate(self, fast=False) -> None: 

836 if self.cowutil.check(self.path, fast=fast) != CowUtil.CheckResult.Success: 836 ↛ 837line 836 didn't jump to line 837, because the condition on line 836 was never true

837 raise util.SMException("COW image %s corrupted" % self) 

838 

839 def _clear(self): 

840 self.uuid = "" 

841 self.path = "" 

842 self.parentUuid = "" 

843 self.parent = None 

844 self._clearRef() 

845 

846 def _clearRef(self): 

847 self._vdiRef = None 

848 

849 @staticmethod 

850 def _cancel_exception(sig, frame): 

851 raise CancelException() 

852 

853 def _call_plugin_coalesce(self, hostRef, leaf): 

854 signal.signal(signal.SIGTERM, self._cancel_exception) 

855 args = {"path": self.path, "vdi_type": self.vdi_type, "leaf_path": leaf.path} 

856 Util.log("Calling remote coalesce plugin with: {}".format(args)) 

857 try: 

858 ret = self.sr.xapi.session.xenapi.host.call_plugin( \ 

859 hostRef, XAPI.PLUGIN_ON_SLAVE, "commit_tapdisk", args) 

860 Util.log("Remote coalesce returned {}".format(ret)) 

861 except CancelException: 

862 Util.log(f"Cancelling online coalesce following signal {args}") 

863 self.sr.xapi.session.xenapi.host.call_plugin( \ 

864 hostRef, XAPI.PLUGIN_ON_SLAVE, "commit_cancel", args) 

865 raise 

866 except Exception: 

867 raise 

868 

869 def _doCoalesceOnHost(self, hostRef, leaf): 

870 self.parent._increaseSizeVirt(self.sizeVirt) 

871 self.sr._updateSlavesOnResize(self.parent) 

872 

873 self._coalesceCowImageOnHost(hostRef, leaf) 

874 

875 #self._verifyContents(0) 

876 self.parent.updateBlockInfo() 

877 

878 def _isOpenOnHosts(self) -> Optional[str]: 

879 for pbdRecord in self.sr.xapi.getAttachedPBDs(): 

880 hostRef = pbdRecord["host"] 

881 args = {"path": self.path} 

882 is_openers = util.strtobool(self.sr.xapi.session.xenapi.host.call_plugin( \ 

883 hostRef, XAPI.PLUGIN_ON_SLAVE, "is_openers", args)) 

884 if is_openers: 

885 return hostRef 

886 return None 

887 

888 def _doCoalesce(self) -> None: 

889 """Coalesce self onto parent. Only perform the actual coalescing of 

890 an image, but not the subsequent relinking. We'll do that as the next step, 

891 after reloading the entire SR in case things have changed while we 

892 were coalescing""" 

893 self.validate() 

894 self.parent.validate(True) 

895 self.parent._increaseSizeVirt(self.sizeVirt) 

896 self.sr._updateSlavesOnResize(self.parent) 

897 self._coalesceCowImage(0) 

898 self.parent.validate(True) 

899 #self._verifyContents(0) 

900 self.parent.updateBlockInfo() 

901 

902 def _verifyContents(self, timeOut): 

903 Util.log(" Coalesce verification on %s" % self) 

904 abortTest = lambda: IPCFlag(self.sr.uuid).test(FLAG_TYPE_ABORT) 

905 Util.runAbortable(lambda: self._runTapdiskDiff(), True, 

906 self.sr.uuid, abortTest, VDI.POLL_INTERVAL, timeOut) 

907 Util.log(" Coalesce verification succeeded") 

908 

909 def _runTapdiskDiff(self): 

910 cmd = "tapdisk-diff -n %s:%s -m %s:%s" % \ 

911 (self.getDriverName(), self.path, \ 

912 self.parent.getDriverName(), self.parent.path) 

913 Util.doexec(cmd, 0) 

914 return True 

915 

916 @staticmethod 

917 def _reportCoalesceError(vdi, ce): 

918 """Reports a coalesce error to XenCenter. 

919 

920 vdi: the VDI object on which the coalesce error occured 

921 ce: the CommandException that was raised""" 

922 

923 msg_name = os.strerror(ce.code) 

924 if ce.code == errno.ENOSPC: 

925 # TODO We could add more information here, e.g. exactly how much 

926 # space is required for the particular coalesce, as well as actions 

927 # to be taken by the user and consequences of not taking these 

928 # actions. 

929 msg_body = 'Run out of space while coalescing.' 

930 elif ce.code == errno.EIO: 

931 msg_body = 'I/O error while coalescing.' 

932 else: 

933 msg_body = '' 

934 util.SMlog('Coalesce failed on SR %s: %s (%s)' 

935 % (vdi.sr.uuid, msg_name, msg_body)) 

936 

937 # Create a XenCenter message, but don't spam. 

938 xapi = vdi.sr.xapi.session.xenapi 

939 sr_ref = xapi.SR.get_by_uuid(vdi.sr.uuid) 

940 oth_cfg = xapi.SR.get_other_config(sr_ref) 

941 if COALESCE_ERR_RATE_TAG in oth_cfg: 

942 coalesce_err_rate = float(oth_cfg[COALESCE_ERR_RATE_TAG]) 

943 else: 

944 coalesce_err_rate = DEFAULT_COALESCE_ERR_RATE 

945 

946 xcmsg = False 

947 if coalesce_err_rate == 0: 

948 xcmsg = True 

949 elif coalesce_err_rate > 0: 

950 now = datetime.datetime.now() 

951 sm_cfg = xapi.SR.get_sm_config(sr_ref) 

952 if COALESCE_LAST_ERR_TAG in sm_cfg: 

953 # seconds per message (minimum distance in time between two 

954 # messages in seconds) 

955 spm = datetime.timedelta(seconds=(1.0 / coalesce_err_rate) * 60) 

956 last = datetime.datetime.fromtimestamp( 

957 float(sm_cfg[COALESCE_LAST_ERR_TAG])) 

958 if now - last >= spm: 

959 xapi.SR.remove_from_sm_config(sr_ref, 

960 COALESCE_LAST_ERR_TAG) 

961 xcmsg = True 

962 else: 

963 xcmsg = True 

964 if xcmsg: 

965 xapi.SR.add_to_sm_config(sr_ref, COALESCE_LAST_ERR_TAG, 

966 str(now.strftime('%s'))) 

967 if xcmsg: 

968 xapi.message.create(msg_name, "3", "SR", vdi.sr.uuid, msg_body) 

969 

970 def coalesce(self) -> int: 

971 return self.cowutil.coalesce(self.path) 

972 

973 @staticmethod 

974 def _doCoalesceCowImage(vdi: "VDI"): 

975 try: 

976 startTime = time.time() 

977 allocated_size = vdi.getAllocatedSize() 

978 coalesced_size = vdi.coalesce() 

979 endTime = time.time() 

980 vdi.sr.recordStorageSpeed(startTime, endTime, coalesced_size) 

981 except util.CommandException as ce: 

982 # We use try/except for the following piece of code because it runs 

983 # in a separate process context and errors will not be caught and 

984 # reported by anyone. 

985 try: 

986 # Report coalesce errors back to user via XC 

987 VDI._reportCoalesceError(vdi, ce) 

988 except Exception as e: 

989 util.SMlog('failed to create XenCenter message: %s' % e) 

990 raise ce 

991 except: 

992 raise 

993 

994 def _vdi_is_raw(self, vdi_path): 

995 """ 

996 Given path to vdi determine if it is raw 

997 """ 

998 uuid = self.extractUuid(vdi_path) 

999 return self.sr.vdis[uuid].vdi_type == VdiType.RAW 

1000 

1001 def _coalesceCowImage(self, timeOut): 

1002 Util.log(" Running COW coalesce on %s" % self) 

1003 def abortTest(): 

1004 if self.cowutil.isCoalesceableOnRemote(): 

1005 file = self.sr._gc_running_file(self) 

1006 try: 

1007 with open(file, "r") as f: 

1008 if not f.read(): 

1009 return True 

1010 except OSError as e: 

1011 if e.errno == errno.ENOENT: 

1012 util.SMlog("File {} does not exist".format(file)) 

1013 else: 

1014 util.SMlog("IOError: {}".format(e)) 

1015 return True 

1016 return IPCFlag(self.sr.uuid).test(FLAG_TYPE_ABORT) 

1017 

1018 try: 

1019 util.fistpoint.activate_custom_fn( 

1020 "cleanup_coalesceVHD_inject_failure", 

1021 util.inject_failure) 

1022 Util.runAbortable(lambda: VDI._doCoalesceCowImage(self), None, 

1023 self.sr.uuid, abortTest, VDI.POLL_INTERVAL, timeOut) 

1024 except: 

1025 # Exception at this phase could indicate a failure in COW coalesce 

1026 # or a kill of COW coalesce by runAbortable due to timeOut 

1027 # Try a repair and reraise the exception 

1028 parent = "" 

1029 try: 

1030 parent = self.getParent() 

1031 if not self._vdi_is_raw(parent): 

1032 # Repair error is logged and ignored. Error reraised later 

1033 util.SMlog('Coalesce failed on %s, attempting repair on ' \ 

1034 'parent %s' % (self.uuid, parent)) 

1035 self.repair(parent) 

1036 except Exception as e: 

1037 util.SMlog('(error ignored) Failed to repair parent %s ' \ 

1038 'after failed coalesce on %s, err: %s' % 

1039 (parent, self.path, e)) 

1040 raise 

1041 

1042 util.fistpoint.activate("LVHDRT_coalescing_VHD_data", self.sr.uuid) 

1043 

1044 def _coalesceCowImageOnHost(self, hostRef, leaf): 

1045 Util.log(" Running COW coalesce on {} via remote host {}".format(self, hostRef)) 

1046 def abortTest(): 

1047 file = self.sr._gc_running_file(self) 

1048 try: 

1049 with open(file, "r") as f: 

1050 if not f.read(): 

1051 Util.log("abortTest: Cancelling coalesce") 

1052 return True 

1053 except OSError as e: 

1054 if e.errno == errno.ENOENT: 

1055 Util.log("File {} does not exist".format(file)) 

1056 else: 

1057 Util.log("IOError: {}".format(e)) 

1058 return True 

1059 return False 

1060 

1061 Util.runAbortable(lambda: self._call_plugin_coalesce(hostRef, leaf), 

1062 None, self.sr.uuid, abortTest, VDI.POLL_INTERVAL, 0, prefSig=signal.SIGTERM) 

1063 

1064 def _relinkSkip(self) -> None: 

1065 """Relink children of this VDI to point to the parent of this VDI""" 

1066 abortFlag = IPCFlag(self.sr.uuid) 

1067 for child in self.children: 

1068 if abortFlag.test(FLAG_TYPE_ABORT): 1068 ↛ 1069line 1068 didn't jump to line 1069, because the condition on line 1068 was never true

1069 raise AbortException("Aborting due to signal") 

1070 Util.log(" Relinking %s from %s to %s" % \ 

1071 (child, self, self.parent)) 

1072 util.fistpoint.activate("LVHDRT_relinking_grandchildren", self.sr.uuid) 

1073 child._setParent(self.parent) 

1074 self.children = [] 

1075 

1076 def _reloadChildren(self, vdiSkip): 

1077 """Pause & unpause all VDIs in the subtree to cause blktap to reload 

1078 the COW image metadata for this file in any online VDI""" 

1079 abortFlag = IPCFlag(self.sr.uuid) 

1080 for child in self.children: 

1081 if child == vdiSkip: 

1082 continue 

1083 if abortFlag.test(FLAG_TYPE_ABORT): 1083 ↛ 1084line 1083 didn't jump to line 1084, because the condition on line 1083 was never true

1084 raise AbortException("Aborting due to signal") 

1085 Util.log(" Reloading VDI %s" % child) 

1086 child._reload() 

1087 

1088 def _reload(self): 

1089 """Pause & unpause to cause blktap to reload the image metadata""" 

1090 for child in self.children: 1090 ↛ 1091line 1090 didn't jump to line 1091, because the loop on line 1090 never started

1091 child._reload() 

1092 

1093 # only leaves can be attached 

1094 if len(self.children) == 0: 1094 ↛ exitline 1094 didn't return from function '_reload', because the condition on line 1094 was never false

1095 try: 

1096 self.delConfig(VDI.DB_VDI_RELINKING) 

1097 except XenAPI.Failure as e: 

1098 if not util.isInvalidVDI(e): 

1099 raise 

1100 self.refresh() 

1101 

1102 def _needRelink(self, list_not_to_relink): 

1103 """ 

1104 If we coalesce up the chain, we shouldn't need to do the relink at all, we only need to do the relink on the children if their direct parent was the one we were coalescing 

1105 """ 

1106 if not list_not_to_relink: 1106 ↛ 1108line 1106 didn't jump to line 1108, because the condition on line 1106 was never false

1107 return True 

1108 if self.uuid in list_not_to_relink: 

1109 return False 

1110 else: 

1111 return True 

1112 

1113 def _tagChildrenForRelink(self, list_not_to_relink=None): 

1114 if len(self.children) == 0: 

1115 retries = 0 

1116 try: 

1117 while retries < 15: 

1118 retries += 1 

1119 if self.getConfig(VDI.DB_VDI_ACTIVATING) is not None: 

1120 Util.log("VDI %s is activating, wait to relink" % 

1121 self.uuid) 

1122 else: 

1123 if self._needRelink(list_not_to_relink): 1123 ↛ 1133line 1123 didn't jump to line 1133, because the condition on line 1123 was never false

1124 self.setConfig(VDI.DB_VDI_RELINKING, "True") 

1125 

1126 if self.getConfig(VDI.DB_VDI_ACTIVATING): 

1127 self.delConfig(VDI.DB_VDI_RELINKING) 

1128 Util.log("VDI %s started activating while tagging" % 

1129 self.uuid) 

1130 else: 

1131 return 

1132 else: 

1133 Util.log(f"Not adding relinking tag to VDI {self.uuid}") 

1134 return 

1135 time.sleep(2) 

1136 

1137 raise util.SMException("Failed to tag vdi %s for relink" % self) 

1138 except XenAPI.Failure as e: 

1139 if not util.isInvalidVDI(e): 

1140 raise 

1141 

1142 for child in self.children: 

1143 child._tagChildrenForRelink(list_not_to_relink) 

1144 

1145 def _loadInfoParent(self): 

1146 ret = self.cowutil.getParent(self.path, LvmCowUtil.extractUuid) 

1147 if ret: 

1148 self.parentUuid = ret 

1149 

1150 def _setParent(self, parent) -> None: 

1151 self.cowutil.setParent(self.path, parent.path, False) 

1152 self.parent = parent 

1153 self.parentUuid = parent.uuid 

1154 parent.children.append(self) 

1155 try: 

1156 self.setConfig(self.DB_VDI_PARENT, self.parentUuid) 

1157 Util.log("Updated the vhd-parent field for child %s with %s" % \ 

1158 (self.uuid, self.parentUuid)) 

1159 except: 

1160 Util.log("Failed to update %s with vhd-parent field %s" % \ 

1161 (self.uuid, self.parentUuid)) 

1162 

1163 def _ensureParentActiveForRelink(self) -> None: 

1164 pass 

1165 

1166 def _update_vhd_parent(self, real_parent_uuid): 

1167 try: 

1168 self.setConfig(self.DB_VDI_PARENT, real_parent_uuid) 

1169 Util.log("Updated the vhd-parent field for child %s with real parent %s following a online coalesce" % \ 

1170 (self.uuid, real_parent_uuid)) 

1171 except: 

1172 Util.log("Failed to update %s with vhd-parent field %s" % \ 

1173 (self.uuid, real_parent_uuid)) 

1174 

1175 def isHidden(self) -> bool: 

1176 if self._hidden is None: 1176 ↛ 1177line 1176 didn't jump to line 1177, because the condition on line 1176 was never true

1177 self._loadInfoHidden() 

1178 return self._hidden 

1179 

1180 def _loadInfoHidden(self) -> None: 

1181 hidden = self.cowutil.getHidden(self.path) 

1182 self._hidden = (hidden != 0) 

1183 

1184 def _setHidden(self, hidden=True) -> None: 

1185 self._hidden = None 

1186 self.cowutil.setHidden(self.path, hidden) 

1187 self._hidden = hidden 

1188 

1189 def _increaseSizeVirt(self, size, atomic=True) -> None: 

1190 """ensure the virtual size of 'self' is at least 'size'. Note that 

1191 resizing a COW image must always be offline and atomically: the file must 

1192 not be open by anyone and no concurrent operations may take place. 

1193 Thus we use the Agent API call for performing paused atomic 

1194 operations. If the caller is already in the atomic context, it must 

1195 call with atomic = False""" 

1196 if self.sizeVirt >= size: 1196 ↛ 1198line 1196 didn't jump to line 1198, because the condition on line 1196 was never false

1197 return 

1198 Util.log(" Expanding COW image virt size for VDI %s: %s -> %s" % \ 

1199 (self, Util.num2str(self.sizeVirt), Util.num2str(size))) 

1200 

1201 msize = self.cowutil.getMaxResizeSize(self.path) 

1202 if (size <= msize): 

1203 self.cowutil.setSizeVirtFast(self.path, size) 

1204 else: 

1205 if atomic: 

1206 vdiList = self._getAllSubtree() 

1207 self.sr.lock() 

1208 try: 

1209 self.sr.pauseVDIs(vdiList) 

1210 try: 

1211 self._setSizeVirt(size) 

1212 finally: 

1213 self.sr.unpauseVDIs(vdiList) 

1214 finally: 

1215 self.sr.unlock() 

1216 else: 

1217 self._setSizeVirt(size) 

1218 

1219 self.sizeVirt = self.cowutil.getSizeVirt(self.path) 

1220 

1221 def _setSizeVirt(self, size) -> None: 

1222 """WARNING: do not call this method directly unless all VDIs in the 

1223 subtree are guaranteed to be unplugged (and remain so for the duration 

1224 of the operation): this operation is only safe for offline COW images""" 

1225 jFile = os.path.join(self.sr.path, self.uuid) 

1226 self.cowutil.setSizeVirt(self.path, size, jFile) 

1227 

1228 def _queryCowBlocks(self) -> bytes: 

1229 return self.cowutil.getBlockBitmap(self.path) 

1230 

1231 def _getCoalescedSizeData(self): 

1232 """Get the data size of the resulting image if we coalesce self onto 

1233 parent. We calculate the actual size by using the image block allocation 

1234 information (as opposed to just adding up the two image sizes to get an 

1235 upper bound)""" 

1236 # make sure we don't use stale BAT info from vdi_rec since the child 

1237 # was writable all this time 

1238 self.delConfig(VDI.DB_VDI_BLOCKS) 

1239 blocksChild = self.getVDIBlocks() 

1240 blocksParent = self.parent.getVDIBlocks() 

1241 numBlocks = Util.countBits(blocksChild, blocksParent) 

1242 Util.log("Num combined blocks = %d" % numBlocks) 

1243 sizeData = numBlocks * self.cowutil.getBlockSize(self.path) 

1244 assert(sizeData <= self.sizeVirt) 

1245 return sizeData 

1246 

1247 def _calcExtraSpaceForCoalescing(self) -> int: 

1248 sizeData = self._getCoalescedSizeData() 

1249 sizeCoalesced = sizeData + self.cowutil.calcOverheadBitmap(sizeData) + \ 

1250 self.cowutil.calcOverheadEmpty(self.sizeVirt) 

1251 Util.log("Coalesced size = %s" % Util.num2str(sizeCoalesced)) 

1252 return sizeCoalesced - self.parent.getSizePhys() 

1253 

1254 def _calcExtraSpaceForLeafCoalescing(self) -> int: 

1255 """How much extra space in the SR will be required to 

1256 [live-]leaf-coalesce this VDI""" 

1257 # the space requirements are the same as for inline coalesce 

1258 return self._calcExtraSpaceForCoalescing() 

1259 

1260 def _calcExtraSpaceForSnapshotCoalescing(self) -> int: 

1261 """How much extra space in the SR will be required to 

1262 snapshot-coalesce this VDI""" 

1263 return self._calcExtraSpaceForCoalescing() + \ 

1264 self.cowutil.calcOverheadEmpty(self.sizeVirt) # extra snap leaf 

1265 

1266 def _getAllSubtree(self): 

1267 """Get self and all VDIs in the subtree of self as a flat list""" 

1268 vdiList = [self] 

1269 for child in self.children: 

1270 vdiList.extend(child._getAllSubtree()) 

1271 return vdiList 

1272 

1273 

1274class FileVDI(VDI): 

1275 """Object representing a VDI in a file-based SR (EXT or NFS)""" 

1276 

1277 @override 

1278 @staticmethod 

1279 def extractUuid(path): 

1280 fileName = os.path.basename(path) 

1281 return os.path.splitext(fileName)[0] 

1282 

1283 def __init__(self, sr, uuid, vdi_type): 

1284 VDI.__init__(self, sr, uuid, vdi_type) 

1285 self.fileName = "%s%s" % (self.uuid, VDI_TYPE_TO_EXTENSION[self.vdi_type]) 

1286 

1287 @override 

1288 def load(self, info=None) -> None: 

1289 if not info: 

1290 if not util.pathexists(self.path): 

1291 raise util.SMException("%s not found" % self.path) 

1292 try: 

1293 info = self.cowutil.getInfo(self.path, self.extractUuid) 

1294 except util.SMException: 

1295 Util.log(" [VDI %s: failed to read COW image metadata]" % self.uuid) 

1296 return 

1297 self.parent = None 

1298 self.children = [] 

1299 self.parentUuid = info.parentUuid 

1300 self.sizeVirt = info.sizeVirt 

1301 self._sizePhys = info.sizePhys 

1302 self._sizeAllocated = info.sizeAllocated 

1303 self._hidden = info.hidden 

1304 self.scanError = False 

1305 self.path = os.path.join(self.sr.path, "%s%s" % \ 

1306 (self.uuid, VDI_TYPE_TO_EXTENSION[self.vdi_type])) 

1307 

1308 @override 

1309 def rename(self, uuid) -> None: 

1310 oldPath = self.path 

1311 VDI.rename(self, uuid) 

1312 self.fileName = "%s%s" % (self.uuid, VDI_TYPE_TO_EXTENSION[self.vdi_type]) 

1313 self.path = os.path.join(self.sr.path, self.fileName) 

1314 assert(not util.pathexists(self.path)) 

1315 Util.log("Renaming %s -> %s" % (oldPath, self.path)) 

1316 os.rename(oldPath, self.path) 

1317 

1318 @override 

1319 def delete(self) -> None: 

1320 if len(self.children) > 0: 1320 ↛ 1321line 1320 didn't jump to line 1321, because the condition on line 1320 was never true

1321 raise util.SMException("VDI %s has children, can't delete" % \ 

1322 self.uuid) 

1323 try: 

1324 self.sr.lock() 

1325 try: 

1326 os.unlink(self.path) 

1327 self.sr.forgetVDI(self.uuid) 

1328 finally: 

1329 self.sr.unlock() 

1330 except OSError: 

1331 raise util.SMException("os.unlink(%s) failed" % self.path) 

1332 VDI.delete(self) 

1333 

1334 @override 

1335 def getAllocatedSize(self) -> int: 

1336 if self._sizeAllocated == -1: 1336 ↛ 1337line 1336 didn't jump to line 1337, because the condition on line 1336 was never true

1337 self._sizeAllocated = self.cowutil.getAllocatedSize(self.path) 

1338 return self._sizeAllocated 

1339 

1340 

1341class LVMVDI(VDI): 

1342 """Object representing a VDI in an LVM SR""" 

1343 

1344 JRN_ZERO = "zero" # journal entry type for zeroing out end of parent 

1345 

1346 @override 

1347 def load(self, info=None) -> None: 

1348 # `info` is always set. `None` default value is only here to match parent method. 

1349 assert info, "No info given to LVMVDI.load" 

1350 self.parent = None 

1351 self.children = [] 

1352 self._sizePhys = -1 

1353 self._sizeAllocated = -1 

1354 self.scanError = info.scanError 

1355 self.sizeLV = info.sizeLV 

1356 self.sizeVirt = info.sizeVirt 

1357 self.fileName = info.lvName 

1358 self.lvActive = info.lvActive 

1359 self.lvOpen = info.lvOpen 

1360 self.lvReadonly = info.lvReadonly 

1361 self._hidden = info.hidden 

1362 self.parentUuid = info.parentUuid 

1363 self.path = os.path.join(self.sr.path, self.fileName) 

1364 self.lvmcowutil = LvmCowUtil(self.cowutil) 

1365 

1366 @override 

1367 @staticmethod 

1368 def extractUuid(path): 

1369 return LvmCowUtil.extractUuid(path) 

1370 

1371 def inflate(self, size): 

1372 """inflate the LV containing the COW image to 'size'""" 

1373 if not VdiType.isCowImage(self.vdi_type): 

1374 return 

1375 self._activate() 

1376 self.sr.lock() 

1377 try: 

1378 self.lvmcowutil.inflate(self.sr.journaler, self.sr.uuid, self.uuid, self.vdi_type, size) 

1379 util.fistpoint.activate("LVHDRT_inflating_the_parent", self.sr.uuid) 

1380 finally: 

1381 self.sr.unlock() 

1382 self.sizeLV = self.sr.lvmCache.getSize(self.fileName) 

1383 self._sizePhys = -1 

1384 self._sizeAllocated = -1 

1385 

1386 def deflate(self): 

1387 """deflate the LV containing the image to minimum""" 

1388 if not VdiType.isCowImage(self.vdi_type): 

1389 return 

1390 self._activate() 

1391 self.sr.lock() 

1392 try: 

1393 self.lvmcowutil.deflate(self.sr.lvmCache, self.fileName, self.getSizePhys()) 

1394 finally: 

1395 self.sr.unlock() 

1396 self.sizeLV = self.sr.lvmCache.getSize(self.fileName) 

1397 self._sizePhys = -1 

1398 self._sizeAllocated = -1 

1399 

1400 def inflateFully(self): 

1401 self.inflate(self.lvmcowutil.calcVolumeSize(self.sizeVirt)) 

1402 

1403 def inflateParentForCoalesce(self): 

1404 """Inflate the parent only as much as needed for the purposes of 

1405 coalescing""" 

1406 if not VdiType.isCowImage(self.parent.vdi_type): 

1407 return 

1408 inc = self._calcExtraSpaceForCoalescing() 

1409 if inc > 0: 

1410 util.fistpoint.activate("LVHDRT_coalescing_before_inflate_grandparent", self.sr.uuid) 

1411 self.parent.inflate(self.parent.sizeLV + inc) 

1412 

1413 @override 

1414 def updateBlockInfo(self) -> Optional[str]: 

1415 if VdiType.isCowImage(self.vdi_type): 

1416 return VDI.updateBlockInfo(self) 

1417 return None 

1418 

1419 @override 

1420 def rename(self, uuid) -> None: 

1421 oldUuid = self.uuid 

1422 oldLVName = self.fileName 

1423 VDI.rename(self, uuid) 

1424 self.fileName = LV_PREFIX[self.vdi_type] + self.uuid 

1425 self.path = os.path.join(self.sr.path, self.fileName) 

1426 assert(not self.sr.lvmCache.checkLV(self.fileName)) 

1427 

1428 self.sr.lvmCache.rename(oldLVName, self.fileName) 

1429 if self.sr.lvActivator.get(oldUuid, False): 

1430 self.sr.lvActivator.replace(oldUuid, self.uuid, self.fileName, False) 

1431 

1432 ns = NS_PREFIX_LVM + self.sr.uuid 

1433 (cnt, bcnt) = RefCounter.check(oldUuid, ns) 

1434 RefCounter.set(self.uuid, cnt, bcnt, ns) 

1435 RefCounter.reset(oldUuid, ns) 

1436 

1437 @override 

1438 def delete(self) -> None: 

1439 if len(self.children) > 0: 

1440 raise util.SMException("VDI %s has children, can't delete" % \ 

1441 self.uuid) 

1442 self.sr.lock() 

1443 try: 

1444 self.sr.lvmCache.remove(self.fileName) 

1445 self.sr.forgetVDI(self.uuid) 

1446 finally: 

1447 self.sr.unlock() 

1448 RefCounter.reset(self.uuid, NS_PREFIX_LVM + self.sr.uuid) 

1449 VDI.delete(self) 

1450 

1451 @override 

1452 def getSizePhys(self) -> int: 

1453 if self._sizePhys == -1: 

1454 self._loadInfoSizePhys() 

1455 return self._sizePhys 

1456 

1457 def _loadInfoSizePhys(self): 

1458 """Get the physical utilization of the COW image file. We do it individually 

1459 (and not using the COW batch scanner) as an optimization: this info is 

1460 relatively expensive and we need it only for VDI's involved in 

1461 coalescing.""" 

1462 if not VdiType.isCowImage(self.vdi_type): 

1463 return 

1464 self._activate() 

1465 self._sizePhys = self.cowutil.getSizePhys(self.path) 

1466 if self._sizePhys <= 0: 

1467 raise util.SMException("phys size of %s = %d" % \ 

1468 (self, self._sizePhys)) 

1469 

1470 @override 

1471 def getAllocatedSize(self) -> int: 

1472 if self._sizeAllocated == -1: 

1473 self._loadInfoSizeAllocated() 

1474 return self._sizeAllocated 

1475 

1476 def _loadInfoSizeAllocated(self): 

1477 """ 

1478 Get the allocated size of the COW volume. 

1479 """ 

1480 if not VdiType.isCowImage(self.vdi_type): 

1481 return 

1482 self._activate() 

1483 self._sizeAllocated = self.cowutil.getAllocatedSize(self.path) 

1484 

1485 @override 

1486 def _loadInfoHidden(self) -> None: 

1487 if not VdiType.isCowImage(self.vdi_type): 

1488 self._hidden = self.sr.lvmCache.getHidden(self.fileName) 

1489 else: 

1490 VDI._loadInfoHidden(self) 

1491 

1492 @override 

1493 def _setHidden(self, hidden=True) -> None: 

1494 if not VdiType.isCowImage(self.vdi_type): 

1495 self._hidden = None 

1496 self.sr.lvmCache.setHidden(self.fileName, hidden) 

1497 self._hidden = hidden 

1498 else: 

1499 VDI._setHidden(self, hidden) 

1500 

1501 @override 

1502 def __str__(self) -> str: 

1503 strType = self.vdi_type 

1504 if self.vdi_type == VdiType.RAW: 

1505 strType = "RAW" 

1506 strHidden = "" 

1507 if self.isHidden(): 

1508 strHidden = "*" 

1509 strSizePhys = "" 

1510 if self._sizePhys > 0: 

1511 strSizePhys = Util.num2str(self._sizePhys) 

1512 strSizeAllocated = "" 

1513 if self._sizeAllocated >= 0: 

1514 strSizeAllocated = Util.num2str(self._sizeAllocated) 

1515 strActive = "n" 

1516 if self.lvActive: 

1517 strActive = "a" 

1518 if self.lvOpen: 

1519 strActive += "o" 

1520 return "%s%s[%s](%s/%s/%s/%s|%s)" % (strHidden, self.uuid[0:8], strType, 

1521 Util.num2str(self.sizeVirt), strSizePhys, strSizeAllocated, 

1522 Util.num2str(self.sizeLV), strActive) 

1523 

1524 @override 

1525 def validate(self, fast=False) -> None: 

1526 if VdiType.isCowImage(self.vdi_type): 

1527 VDI.validate(self, fast) 

1528 

1529 def _setChainRw(self) -> List[str]: 

1530 """ 

1531 Set the readonly LV and children writable. 

1532 It's needed because the coalesce can be done by tapdisk directly 

1533 and it will need to write parent information for children. 

1534 The VDI we want to coalesce into it's parent need to be writable for libqcow coalesce part. 

1535 Return a list of the LV that were previously readonly to be made RO again after the coalesce. 

1536 """ 

1537 was_ro = [] 

1538 if self.lvReadonly: 

1539 self.sr.lvmCache.setReadonly(self.fileName, False) 

1540 was_ro.append(self.fileName) 

1541 

1542 for child in self.children: 

1543 if child.lvReadonly: 

1544 self.sr.lvmCache.setReadonly(child.fileName, False) 

1545 was_ro.append(child.fileName) 

1546 

1547 return was_ro 

1548 

1549 def _setChainRo(self, was_ro: List[str]) -> None: 

1550 """Set the list of LV in parameters to readonly""" 

1551 for lvName in was_ro: 

1552 self.sr.lvmCache.setReadonly(lvName, True) 

1553 

1554 @override 

1555 def _doCoalesce(self) -> None: 

1556 """LVMVDI parents must first be activated, inflated, and made writable""" 

1557 was_ro = [] 

1558 try: 

1559 self._activateChain() 

1560 self.sr.lvmCache.setReadonly(self.parent.fileName, False) 

1561 self.parent.validate() 

1562 self.inflateParentForCoalesce() 

1563 was_ro = self._setChainRw() 

1564 VDI._doCoalesce(self) 

1565 finally: 

1566 self.parent._loadInfoSizePhys() 

1567 self.parent.deflate() 

1568 self.sr.lvmCache.setReadonly(self.parent.fileName, True) 

1569 self._setChainRo(was_ro) 

1570 

1571 @override 

1572 def _setParent(self, parent) -> None: 

1573 self._activate() 

1574 if self.lvReadonly: 

1575 self.sr.lvmCache.setReadonly(self.fileName, False) 

1576 

1577 try: 

1578 self.cowutil.setParent(self.path, parent.path, parent.vdi_type == VdiType.RAW) 

1579 finally: 

1580 if self.lvReadonly: 

1581 self.sr.lvmCache.setReadonly(self.fileName, True) 

1582 self._deactivate() 

1583 self.parent = parent 

1584 self.parentUuid = parent.uuid 

1585 parent.children.append(self) 

1586 try: 

1587 self.setConfig(self.DB_VDI_PARENT, self.parentUuid) 

1588 Util.log("Updated the VDI-parent field for child %s with %s" % \ 

1589 (self.uuid, self.parentUuid)) 

1590 except: 

1591 Util.log("Failed to update the VDI-parent with %s for child %s" % \ 

1592 (self.parentUuid, self.uuid)) 

1593 

1594 def _activate(self): 

1595 self.sr.lvActivator.activate(self.uuid, self.fileName, False) 

1596 

1597 def _activateChain(self): 

1598 vdi = self 

1599 while vdi: 

1600 vdi._activate() 

1601 vdi = vdi.parent 

1602 

1603 def _deactivate(self): 

1604 self.sr.lvActivator.deactivate(self.uuid, False) 

1605 

1606 @override 

1607 def _ensureParentActiveForRelink(self) -> None: 

1608 self.parent._activate() 

1609 

1610 @override 

1611 def _increaseSizeVirt(self, size, atomic=True) -> None: 

1612 "ensure the virtual size of 'self' is at least 'size'" 

1613 self._activate() 

1614 if VdiType.isCowImage(self.vdi_type): 

1615 VDI._increaseSizeVirt(self, size, atomic) 

1616 return 

1617 

1618 # raw VDI case 

1619 offset = self.sizeLV 

1620 if self.sizeVirt < size: 

1621 oldSize = self.sizeLV 

1622 self.sizeLV = util.roundup(lvutil.LVM_SIZE_INCREMENT, size) 

1623 Util.log(" Growing %s: %d->%d" % (self.path, oldSize, self.sizeLV)) 

1624 self.sr.lvmCache.setSize(self.fileName, self.sizeLV) 

1625 offset = oldSize 

1626 unfinishedZero = False 

1627 jval = self.sr.journaler.get(self.JRN_ZERO, self.uuid) 

1628 if jval: 

1629 unfinishedZero = True 

1630 offset = int(jval) 

1631 length = self.sizeLV - offset 

1632 if not length: 

1633 return 

1634 

1635 if unfinishedZero: 

1636 Util.log(" ==> Redoing unfinished zeroing out") 

1637 else: 

1638 self.sr.journaler.create(self.JRN_ZERO, self.uuid, \ 

1639 str(offset)) 

1640 Util.log(" Zeroing %s: from %d, %dB" % (self.path, offset, length)) 

1641 abortTest = lambda: IPCFlag(self.sr.uuid).test(FLAG_TYPE_ABORT) 

1642 func = lambda: util.zeroOut(self.path, offset, length) 

1643 Util.runAbortable(func, True, self.sr.uuid, abortTest, 

1644 VDI.POLL_INTERVAL, 0) 

1645 self.sr.journaler.remove(self.JRN_ZERO, self.uuid) 

1646 

1647 @override 

1648 def _setSizeVirt(self, size) -> None: 

1649 """WARNING: do not call this method directly unless all VDIs in the 

1650 subtree are guaranteed to be unplugged (and remain so for the duration 

1651 of the operation): this operation is only safe for offline COW images.""" 

1652 self._activate() 

1653 jFile = self.lvmcowutil.createResizeJournal(self.sr.lvmCache, self.uuid) 

1654 try: 

1655 self.lvmcowutil.setSizeVirt(self.sr.journaler, self.sr.uuid, self.uuid, self.vdi_type, size, jFile) 

1656 finally: 

1657 self.lvmcowutil.destroyResizeJournal(self.sr.lvmCache, self.uuid) 

1658 

1659 @override 

1660 def _queryCowBlocks(self) -> bytes: 

1661 self._activate() 

1662 return VDI._queryCowBlocks(self) 

1663 

1664 @override 

1665 def getParent(self) -> str: 

1666 self._activate() 

1667 parent = VDI.getParent(self) 

1668 self._deactivate() 

1669 return parent 

1670 

1671 @override 

1672 def _calcExtraSpaceForCoalescing(self) -> int: 

1673 if not VdiType.isCowImage(self.parent.vdi_type): 

1674 return 0 # raw parents are never deflated in the first place 

1675 sizeCoalesced = self.lvmcowutil.calcVolumeSize(self._getCoalescedSizeData()) 

1676 Util.log("Coalesced size = %s" % Util.num2str(sizeCoalesced)) 

1677 return sizeCoalesced - self.parent.sizeLV 

1678 

1679 @override 

1680 def _calcExtraSpaceForLeafCoalescing(self) -> int: 

1681 """How much extra space in the SR will be required to 

1682 [live-]leaf-coalesce this VDI""" 

1683 # we can deflate the leaf to minimize the space requirements 

1684 deflateDiff = self.sizeLV - lvutil.calcSizeLV(self.getSizePhys()) 

1685 return self._calcExtraSpaceForCoalescing() - deflateDiff 

1686 

1687 @override 

1688 def _calcExtraSpaceForSnapshotCoalescing(self) -> int: 

1689 return self._calcExtraSpaceForCoalescing() + \ 

1690 lvutil.calcSizeLV(self.getSizePhys()) 

1691 

1692 

1693class LinstorVDI(VDI): 

1694 """Object representing a VDI in a LINSTOR SR""" 

1695 

1696 VOLUME_LOCK_TIMEOUT = 30 

1697 

1698 @override 

1699 def load(self, info=None) -> None: 

1700 self.parentUuid = info.parentUuid 

1701 self.scanError = True 

1702 self.parent = None 

1703 self.children = [] 

1704 

1705 self.fileName = self.sr._linstor.get_volume_name(self.uuid) 

1706 self.path = self.sr._linstor.build_device_path(self.fileName) 

1707 self.linstorcowutil = LinstorCowUtil(self.sr.xapi.session, self.sr._linstor, info.vdiType) 

1708 

1709 if not info: 

1710 try: 

1711 info = self.linstorcowutil.get_info(self.uuid) 

1712 except util.SMException: 

1713 Util.log( 

1714 ' [VDI {}: failed to read COW image metadata]'.format(self.uuid) 

1715 ) 

1716 return 

1717 

1718 self.parentUuid = info.parentUuid 

1719 self.sizeVirt = info.sizeVirt 

1720 self._sizePhys = -1 

1721 self._sizeAllocated = -1 

1722 self.drbd_size = -1 

1723 self._hidden = info.hidden 

1724 self.scanError = False 

1725 

1726 @override 

1727 def getSizePhys(self, fetch=False) -> int: 

1728 if self._sizePhys < 0 or fetch: 

1729 self._sizePhys = self.linstorcowutil.get_size_phys(self.uuid) 

1730 return self._sizePhys 

1731 

1732 def getDrbdSize(self, fetch=False): 

1733 if self.drbd_size < 0 or fetch: 

1734 self.drbd_size = self.linstorcowutil.get_drbd_size(self.uuid) 

1735 return self.drbd_size 

1736 

1737 @override 

1738 def getAllocatedSize(self) -> int: 

1739 if self._sizeAllocated == -1: 

1740 if VdiType.isCowImage(self.vdi_type): 

1741 self._sizeAllocated = self.linstorcowutil.get_allocated_size(self.uuid) 

1742 return self._sizeAllocated 

1743 

1744 def inflate(self, size): 

1745 if not VdiType.isCowImage(self.vdi_type): 

1746 return 

1747 self.sr.lock() 

1748 try: 

1749 # Ensure we use the real DRBD size and not the cached one. 

1750 # Why? Because this attribute can be changed if volume is resized by user. 

1751 self.drbd_size = self.getDrbdSize(fetch=True) 

1752 self.linstorcowutil.inflate(self.sr.journaler, self.uuid, self.path, size, self.drbd_size) 

1753 finally: 

1754 self.sr.unlock() 

1755 self.drbd_size = -1 

1756 self._sizePhys = -1 

1757 self._sizeAllocated = -1 

1758 

1759 def deflate(self): 

1760 if not VdiType.isCowImage(self.vdi_type): 

1761 return 

1762 self.sr.lock() 

1763 try: 

1764 # Ensure we use the real sizes and not the cached info. 

1765 self.drbd_size = self.getDrbdSize(fetch=True) 

1766 self._sizePhys = self.getSizePhys(fetch=True) 

1767 self.linstorcowutil.force_deflate(self.path, self._sizePhys, self.drbd_size, zeroize=False) 

1768 finally: 

1769 self.sr.unlock() 

1770 self.drbd_size = -1 

1771 self._sizePhys = -1 

1772 self._sizeAllocated = -1 

1773 

1774 def inflateFully(self): 

1775 if VdiType.isCowImage(self.vdi_type): 

1776 self.inflate(self.linstorcowutil.compute_volume_size(self.sizeVirt)) 

1777 

1778 @override 

1779 def rename(self, uuid) -> None: 

1780 Util.log('Renaming {} -> {} (path={})'.format( 

1781 self.uuid, uuid, self.path 

1782 )) 

1783 self.sr._linstor.update_volume_uuid(self.uuid, uuid) 

1784 VDI.rename(self, uuid) 

1785 

1786 @override 

1787 def delete(self) -> None: 

1788 if len(self.children) > 0: 

1789 raise util.SMException( 

1790 'VDI {} has children, can\'t delete'.format(self.uuid) 

1791 ) 

1792 self.sr.lock() 

1793 try: 

1794 self.sr._linstor.destroy_volume(self.uuid) 

1795 self.sr.forgetVDI(self.uuid) 

1796 finally: 

1797 self.sr.unlock() 

1798 VDI.delete(self) 

1799 

1800 @override 

1801 def validate(self, fast=False) -> None: 

1802 if VdiType.isCowImage(self.vdi_type) and self.linstorcowutil.check(self.uuid, fast=fast) != CowUtil.CheckResult.Success: 

1803 raise util.SMException('COW image {} corrupted'.format(self)) 

1804 

1805 @override 

1806 def pause(self, failfast=False) -> None: 

1807 self.sr._linstor.ensure_volume_is_not_locked( 

1808 self.uuid, timeout=self.VOLUME_LOCK_TIMEOUT 

1809 ) 

1810 return super(LinstorVDI, self).pause(failfast) 

1811 

1812 @override 

1813 def coalesce(self) -> int: 

1814 # Note: We raise `SMException` here to skip the current coalesce in case of failure. 

1815 # Using another exception we can't execute the next coalesce calls. 

1816 return self.linstorcowutil.force_coalesce(self.path) 

1817 

1818 @override 

1819 def getParent(self) -> str: 

1820 return self.linstorcowutil.get_parent( 

1821 self.sr._linstor.get_volume_uuid_from_device_path(self.path) 

1822 ) 

1823 

1824 @override 

1825 def repair(self, parent_uuid) -> None: 

1826 self.linstorcowutil.force_repair( 

1827 self.sr._linstor.get_device_path(parent_uuid) 

1828 ) 

1829 

1830 @override 

1831 def _relinkSkip(self) -> None: 

1832 abortFlag = IPCFlag(self.sr.uuid) 

1833 for child in self.children: 

1834 if abortFlag.test(FLAG_TYPE_ABORT): 

1835 raise AbortException('Aborting due to signal') 

1836 Util.log( 

1837 ' Relinking {} from {} to {}'.format( 

1838 child, self, self.parent 

1839 ) 

1840 ) 

1841 

1842 session = child.sr.xapi.session 

1843 sr_uuid = child.sr.uuid 

1844 vdi_uuid = child.uuid 

1845 try: 

1846 self.sr._linstor.ensure_volume_is_not_locked( 

1847 vdi_uuid, timeout=self.VOLUME_LOCK_TIMEOUT 

1848 ) 

1849 blktap2.VDI.tap_pause(session, sr_uuid, vdi_uuid) 

1850 child._setParent(self.parent) 

1851 finally: 

1852 blktap2.VDI.tap_unpause(session, sr_uuid, vdi_uuid) 

1853 self.children = [] 

1854 

1855 @override 

1856 def _setParent(self, parent) -> None: 

1857 self.sr._linstor.get_device_path(self.uuid) 

1858 self.linstorcowutil.force_parent(self.path, parent.path) 

1859 self.parent = parent 

1860 self.parentUuid = parent.uuid 

1861 parent.children.append(self) 

1862 try: 

1863 self.setConfig(self.DB_VDI_PARENT, self.parentUuid) 

1864 Util.log("Updated the vhd-parent field for child %s with %s" % \ 

1865 (self.uuid, self.parentUuid)) 

1866 except: 

1867 Util.log("Failed to update %s with vhd-parent field %s" % \ 

1868 (self.uuid, self.parentUuid)) 

1869 

1870 @override 

1871 def _doCoalesce(self) -> None: 

1872 try: 

1873 self._activateChain() 

1874 self.parent.validate() 

1875 self._inflateParentForCoalesce() 

1876 VDI._doCoalesce(self) 

1877 finally: 

1878 self.parent.deflate() 

1879 

1880 def _activateChain(self): 

1881 vdi = self 

1882 while vdi: 

1883 try: 

1884 p = self.sr._linstor.get_device_path(vdi.uuid) 

1885 except Exception as e: 

1886 # Use SMException to skip coalesce. 

1887 # Otherwise the GC is stopped... 

1888 raise util.SMException(str(e)) 

1889 vdi = vdi.parent 

1890 

1891 @override 

1892 def _setHidden(self, hidden=True) -> None: 

1893 HIDDEN_TAG = 'hidden' 

1894 

1895 if not VdiType.isCowImage(self.vdi_type): 

1896 self._hidden = None 

1897 self.sr._linstor.update_volume_metadata(self.uuid, { 

1898 HIDDEN_TAG: hidden 

1899 }) 

1900 self._hidden = hidden 

1901 else: 

1902 VDI._setHidden(self, hidden) 

1903 

1904 @override 

1905 def _increaseSizeVirt(self, size, atomic=True): 

1906 if self.vdi_type == VdiType.RAW: 

1907 offset = self.drbd_size 

1908 if self.sizeVirt < size: 

1909 oldSize = self.drbd_size 

1910 self.drbd_size = LinstorVolumeManager.round_up_volume_size(size) 

1911 Util.log(" Growing %s: %d->%d" % (self.path, oldSize, self.drbd_size)) 

1912 self.sr._linstor.resize_volume(self.uuid, self.drbd_size) 

1913 offset = oldSize 

1914 unfinishedZero = False 

1915 jval = self.sr.journaler.get(LinstorJournaler.ZERO, self.uuid) 

1916 if jval: 

1917 unfinishedZero = True 

1918 offset = int(jval) 

1919 length = self.drbd_size - offset 

1920 if not length: 

1921 return 

1922 

1923 if unfinishedZero: 

1924 Util.log(" ==> Redoing unfinished zeroing out") 

1925 else: 

1926 self.sr.journaler.create(LinstorJournaler.ZERO, self.uuid, str(offset)) 

1927 Util.log(" Zeroing %s: from %d, %dB" % (self.path, offset, length)) 

1928 abortTest = lambda: IPCFlag(self.sr.uuid).test(FLAG_TYPE_ABORT) 

1929 func = lambda: util.zeroOut(self.path, offset, length) 

1930 Util.runAbortable(func, True, self.sr.uuid, abortTest, VDI.POLL_INTERVAL, 0) 

1931 self.sr.journaler.remove(LinstorJournaler.ZERO, self.uuid) 

1932 return 

1933 

1934 if self.sizeVirt >= size: 

1935 return 

1936 Util.log(" Expanding COW image virt size for VDI %s: %s -> %s" % \ 

1937 (self, Util.num2str(self.sizeVirt), Util.num2str(size))) 

1938 

1939 msize = self.linstorcowutil.get_max_resize_size(self.uuid) * 1024 * 1024 

1940 if (size <= msize): 

1941 self.linstorcowutil.set_size_virt_fast(self.path, size) 

1942 else: 

1943 if atomic: 

1944 vdiList = self._getAllSubtree() 

1945 self.sr.lock() 

1946 try: 

1947 self.sr.pauseVDIs(vdiList) 

1948 try: 

1949 self._setSizeVirt(size) 

1950 finally: 

1951 self.sr.unpauseVDIs(vdiList) 

1952 finally: 

1953 self.sr.unlock() 

1954 else: 

1955 self._setSizeVirt(size) 

1956 

1957 self.sizeVirt = self.linstorcowutil.get_size_virt(self.uuid) 

1958 

1959 @override 

1960 def _setSizeVirt(self, size) -> None: 

1961 jfile = self.uuid + '-jvhd' 

1962 self.sr._linstor.create_volume( 

1963 jfile, self.cowutil.getResizeJournalSize(), persistent=False, volume_name=jfile 

1964 ) 

1965 try: 

1966 self.inflate(self.linstorcowutil.compute_volume_size(size)) 

1967 self.linstorcowutil.set_size_virt(self.path, size, jfile) 

1968 finally: 

1969 try: 

1970 self.sr._linstor.destroy_volume(jfile) 

1971 except Exception: 

1972 # We can ignore it, in any case this volume is not persistent. 

1973 pass 

1974 

1975 @override 

1976 def _queryCowBlocks(self) -> bytes: 

1977 return self.linstorcowutil.get_block_bitmap(self.uuid) 

1978 

1979 def _inflateParentForCoalesce(self): 

1980 if not VdiType.isCowImage(self.parent.vdi_type): 

1981 return 

1982 inc = self._calcExtraSpaceForCoalescing() 

1983 if inc > 0: 

1984 self.parent.inflate(self.parent.getDrbdSize() + inc) 

1985 

1986 @override 

1987 def _calcExtraSpaceForCoalescing(self) -> int: 

1988 if not VdiType.isCowImage(self.parent.vdi_type): 

1989 return 0 

1990 size_coalesced = self.linstorcowutil.compute_volume_size(self._getCoalescedSizeData()) 

1991 Util.log("Coalesced size = %s" % Util.num2str(size_coalesced)) 

1992 return size_coalesced - self.parent.getDrbdSize() 

1993 

1994 @override 

1995 def _calcExtraSpaceForLeafCoalescing(self) -> int: 

1996 assert self.getDrbdSize() > 0 

1997 assert self.getSizePhys() > 0 

1998 deflate_diff = self.getDrbdSize() - LinstorVolumeManager.round_up_volume_size(self.getSizePhys()) 

1999 assert deflate_diff >= 0 

2000 return self._calcExtraSpaceForCoalescing() - deflate_diff 

2001 

2002 @override 

2003 def _calcExtraSpaceForSnapshotCoalescing(self) -> int: 

2004 assert self.getSizePhys() > 0 

2005 return self._calcExtraSpaceForCoalescing() + \ 

2006 LinstorVolumeManager.round_up_volume_size(self.getSizePhys()) 

2007 

2008################################################################################ 

2009# 

2010# SR 

2011# 

2012class SR(object): 

2013 class LogFilter: 

2014 def __init__(self, sr): 

2015 self.sr = sr 

2016 self.stateLogged = False 

2017 self.prevState = {} 

2018 self.currState = {} 

2019 

2020 def logState(self): 

2021 changes = "" 

2022 self.currState.clear() 

2023 for vdi in self.sr.vdiTrees: 

2024 self.currState[vdi.uuid] = self._getTreeStr(vdi) 

2025 if not self.prevState.get(vdi.uuid) or \ 

2026 self.prevState[vdi.uuid] != self.currState[vdi.uuid]: 

2027 changes += self.currState[vdi.uuid] 

2028 

2029 for uuid in self.prevState: 

2030 if not self.currState.get(uuid): 

2031 changes += "Tree %s gone\n" % uuid 

2032 

2033 result = "SR %s (%d VDIs in %d COW trees): " % \ 

2034 (self.sr, len(self.sr.vdis), len(self.sr.vdiTrees)) 

2035 

2036 if len(changes) > 0: 

2037 if self.stateLogged: 

2038 result += "showing only COW trees that changed:" 

2039 result += "\n%s" % changes 

2040 else: 

2041 result += "no changes" 

2042 

2043 for line in result.split("\n"): 

2044 Util.log("%s" % line) 

2045 self.prevState.clear() 

2046 for key, val in self.currState.items(): 

2047 self.prevState[key] = val 

2048 self.stateLogged = True 

2049 

2050 def logNewVDI(self, uuid): 

2051 if self.stateLogged: 

2052 Util.log("Found new VDI when scanning: %s" % uuid) 

2053 

2054 def _getTreeStr(self, vdi, indent=8): 

2055 treeStr = "%s%s\n" % (" " * indent, vdi) 

2056 for child in vdi.children: 

2057 treeStr += self._getTreeStr(child, indent + VDI.STR_TREE_INDENT) 

2058 return treeStr 

2059 

2060 TYPE_FILE = "file" 

2061 TYPE_LVHD = "lvhd" 

2062 TYPE_LINSTOR = "linstor" 

2063 TYPES = [TYPE_LVHD, TYPE_FILE, TYPE_LINSTOR] 

2064 

2065 LOCK_RETRY_INTERVAL = 3 

2066 LOCK_RETRY_ATTEMPTS = 20 

2067 LOCK_RETRY_ATTEMPTS_LOCK = 100 

2068 

2069 SCAN_RETRY_ATTEMPTS = 3 

2070 

2071 JRN_CLONE = "clone" # journal entry type for the clone operation (from SM) 

2072 TMP_RENAME_PREFIX = "OLD_" 

2073 

2074 KEY_OFFLINE_COALESCE_NEEDED = "leaf_coalesce_need_offline" 

2075 KEY_OFFLINE_COALESCE_OVERRIDE = "leaf_coalesce_offline_override" 

2076 

2077 @staticmethod 

2078 def getInstance(uuid, xapiSession, createLock=True, force=False): 

2079 xapi = XAPI(xapiSession, uuid) 

2080 type = normalizeType(xapi.srRecord["type"]) 

2081 if type == SR.TYPE_FILE: 

2082 return FileSR(uuid, xapi, createLock, force) 

2083 elif type == SR.TYPE_LVHD: 

2084 return LVMSR(uuid, xapi, createLock, force) 

2085 elif type == SR.TYPE_LINSTOR: 

2086 return LinstorSR(uuid, xapi, createLock, force) 

2087 raise util.SMException("SR type %s not recognized" % type) 

2088 

2089 def __init__(self, uuid, xapi, createLock, force): 

2090 self.logFilter = self.LogFilter(self) 

2091 self.uuid = uuid 

2092 self.path = "" 

2093 self.name = "" 

2094 self.vdis = {} 

2095 self.vdiTrees = [] 

2096 self.journaler = None 

2097 self.xapi = xapi 

2098 self._locked = 0 

2099 self._srLock = None 

2100 if createLock: 2100 ↛ 2101line 2100 didn't jump to line 2101, because the condition on line 2100 was never true

2101 self._srLock = lock.Lock(lock.LOCK_TYPE_SR, self.uuid) 

2102 else: 

2103 Util.log("Requested no SR locking") 

2104 self.name = self.xapi.srRecord["name_label"] 

2105 self._failedCoalesceTargets = [] 

2106 

2107 if not self.xapi.isPluggedHere(): 

2108 if force: 2108 ↛ 2109line 2108 didn't jump to line 2109, because the condition on line 2108 was never true

2109 Util.log("SR %s not attached on this host, ignoring" % uuid) 

2110 else: 

2111 if not self.wait_for_plug(): 

2112 raise util.SMException("SR %s not attached on this host" % uuid) 

2113 

2114 if force: 2114 ↛ 2115line 2114 didn't jump to line 2115, because the condition on line 2114 was never true

2115 Util.log("Not checking if we are Master (SR %s)" % uuid) 

2116 elif not self.xapi.isMaster(): 2116 ↛ 2117line 2116 didn't jump to line 2117, because the condition on line 2116 was never true

2117 raise util.SMException("This host is NOT master, will not run") 

2118 

2119 self.no_space_candidates = {} 

2120 

2121 def msg_cleared(self, xapi_session, msg_ref): 

2122 try: 

2123 msg = xapi_session.xenapi.message.get_record(msg_ref) 

2124 except XenAPI.Failure: 

2125 return True 

2126 

2127 return msg is None 

2128 

2129 def check_no_space_candidates(self): 

2130 xapi_session = self.xapi.getSession() 

2131 

2132 msg_id = self.xapi.srRecord["sm_config"].get(VDI.DB_GC_NO_SPACE) 

2133 if self.no_space_candidates: 

2134 if msg_id is None or self.msg_cleared(xapi_session, msg_id): 

2135 util.SMlog("Could not coalesce due to a lack of space " 

2136 f"in SR {self.uuid}") 

2137 msg_body = ("Unable to perform data coalesce due to a lack " 

2138 f"of space in SR {self.uuid}") 

2139 msg_id = xapi_session.xenapi.message.create( 

2140 'SM_GC_NO_SPACE', 

2141 3, 

2142 "SR", 

2143 self.uuid, 

2144 msg_body) 

2145 xapi_session.xenapi.SR.remove_from_sm_config( 

2146 self.xapi.srRef, VDI.DB_GC_NO_SPACE) 

2147 xapi_session.xenapi.SR.add_to_sm_config( 

2148 self.xapi.srRef, VDI.DB_GC_NO_SPACE, msg_id) 

2149 

2150 for candidate in self.no_space_candidates.values(): 

2151 candidate.setConfig(VDI.DB_GC_NO_SPACE, msg_id) 

2152 elif msg_id is not None: 

2153 # Everything was coalescable, remove the message 

2154 xapi_session.xenapi.SR.remove_from_sm_config(self.xapi.srRef, VDI.DB_GC_NO_SPACE) 

2155 xapi_session.xenapi.message.destroy(msg_id) 

2156 

2157 def clear_no_space_msg(self, vdi): 

2158 msg_id = None 

2159 try: 

2160 msg_id = vdi.getConfig(VDI.DB_GC_NO_SPACE) 

2161 except XenAPI.Failure: 

2162 pass 

2163 

2164 self.no_space_candidates.pop(vdi.uuid, None) 

2165 if msg_id is not None: 2165 ↛ exitline 2165 didn't return from function 'clear_no_space_msg', because the condition on line 2165 was never false

2166 vdi.delConfig(VDI.DB_GC_NO_SPACE) 

2167 

2168 

2169 def wait_for_plug(self): 

2170 for _ in range(1, 10): 

2171 time.sleep(2) 

2172 if self.xapi.isPluggedHere(): 

2173 return True 

2174 return False 

2175 

2176 def gcEnabled(self, refresh=True): 

2177 if refresh: 

2178 self.xapi.srRecord = \ 

2179 self.xapi.session.xenapi.SR.get_record(self.xapi._srRef) 

2180 if self.xapi.srRecord["other_config"].get(VDI.DB_GC) == "false": 

2181 Util.log("GC is disabled for this SR, abort") 

2182 return False 

2183 return True 

2184 

2185 def scan(self, force=False) -> None: 

2186 """Scan the SR and load VDI info for each VDI. If called repeatedly, 

2187 update VDI objects if they already exist""" 

2188 pass 

2189 

2190 def scanLocked(self, force=False): 

2191 self.lock() 

2192 try: 

2193 self.scan(force) 

2194 finally: 

2195 self.unlock() 

2196 

2197 def getVDI(self, uuid): 

2198 return self.vdis.get(uuid) 

2199 

2200 def hasWork(self): 

2201 if len(self.findGarbage()) > 0: 

2202 return True 

2203 if self.findCoalesceable(): 

2204 return True 

2205 if self.findLeafCoalesceable(): 

2206 return True 

2207 if self.needUpdateBlockInfo(): 

2208 return True 

2209 return False 

2210 

2211 def findCoalesceable(self): 

2212 """Find a coalesceable VDI. Return a vdi that should be coalesced 

2213 (choosing one among all coalesceable candidates according to some 

2214 criteria) or None if there is no VDI that could be coalesced""" 

2215 

2216 candidates = [] 

2217 

2218 srSwitch = self.xapi.srRecord["other_config"].get(VDI.DB_COALESCE) 

2219 if srSwitch == "false": 

2220 Util.log("Coalesce disabled for this SR") 

2221 return candidates 

2222 

2223 # finish any VDI for which a relink journal entry exists first 

2224 journals = self.journaler.getAll(VDI.JRN_RELINK) 

2225 for uuid in journals: 

2226 vdi = self.getVDI(uuid) 

2227 if vdi and vdi not in self._failedCoalesceTargets: 

2228 return vdi 

2229 

2230 for vdi in self.vdis.values(): 

2231 if vdi.isCoalesceable() and vdi not in self._failedCoalesceTargets: 

2232 candidates.append(vdi) 

2233 Util.log("%s is coalescable" % vdi.uuid) 

2234 

2235 self.xapi.update_task_progress("coalescable", len(candidates)) 

2236 

2237 # pick one in the tallest tree 

2238 treeHeight = dict() 

2239 for c in candidates: 

2240 height = c.getTreeRoot().getTreeHeight() 

2241 if treeHeight.get(height): 

2242 treeHeight[height].append(c) 

2243 else: 

2244 treeHeight[height] = [c] 

2245 

2246 freeSpace = self.getFreeSpace() 

2247 heights = list(treeHeight.keys()) 

2248 heights.sort(reverse=True) 

2249 for h in heights: 

2250 for c in treeHeight[h]: 

2251 spaceNeeded = c._calcExtraSpaceForCoalescing() 

2252 if spaceNeeded <= freeSpace: 

2253 Util.log("Coalesce candidate: %s (tree height %d)" % (c, h)) 

2254 self.clear_no_space_msg(c) 

2255 return c 

2256 else: 

2257 self.no_space_candidates[c.uuid] = c 

2258 Util.log("No space to coalesce %s (free space: %d)" % \ 

2259 (c, freeSpace)) 

2260 return None 

2261 

2262 def getSwitch(self, key): 

2263 return self.xapi.srRecord["other_config"].get(key) 

2264 

2265 def forbiddenBySwitch(self, switch, condition, fail_msg): 

2266 srSwitch = self.getSwitch(switch) 

2267 ret = False 

2268 if srSwitch: 

2269 ret = srSwitch == condition 

2270 

2271 if ret: 

2272 Util.log(fail_msg) 

2273 

2274 return ret 

2275 

2276 def leafCoalesceForbidden(self): 

2277 return (self.forbiddenBySwitch(VDI.DB_COALESCE, 

2278 "false", 

2279 "Coalesce disabled for this SR") or 

2280 self.forbiddenBySwitch(VDI.DB_LEAFCLSC, 

2281 VDI.LEAFCLSC_DISABLED, 

2282 "Leaf-coalesce disabled for this SR")) 

2283 

2284 def findLeafCoalesceable(self): 

2285 """Find leaf-coalesceable VDIs in each COW tree""" 

2286 

2287 candidates = [] 

2288 if self.leafCoalesceForbidden(): 

2289 return candidates 

2290 

2291 self.gatherLeafCoalesceable(candidates) 

2292 

2293 self.xapi.update_task_progress("coalescable", len(candidates)) 

2294 

2295 freeSpace = self.getFreeSpace() 

2296 for candidate in candidates: 

2297 # check the space constraints to see if leaf-coalesce is actually 

2298 # feasible for this candidate 

2299 spaceNeeded = candidate._calcExtraSpaceForSnapshotCoalescing() 

2300 spaceNeededLive = spaceNeeded 

2301 if spaceNeeded > freeSpace: 

2302 spaceNeededLive = candidate._calcExtraSpaceForLeafCoalescing() 

2303 if candidate.canLiveCoalesce(self.getStorageSpeed()): 

2304 spaceNeeded = spaceNeededLive 

2305 

2306 if spaceNeeded <= freeSpace: 

2307 Util.log("Leaf-coalesce candidate: %s" % candidate) 

2308 self.clear_no_space_msg(candidate) 

2309 return candidate 

2310 else: 

2311 Util.log("No space to leaf-coalesce %s (free space: %d)" % \ 

2312 (candidate, freeSpace)) 

2313 if spaceNeededLive <= freeSpace: 

2314 Util.log("...but enough space if skip snap-coalesce") 

2315 candidate.setConfig(VDI.DB_LEAFCLSC, 

2316 VDI.LEAFCLSC_OFFLINE) 

2317 self.no_space_candidates[candidate.uuid] = candidate 

2318 

2319 return None 

2320 

2321 def gatherLeafCoalesceable(self, candidates): 

2322 for vdi in self.vdis.values(): 

2323 if not vdi.isLeafCoalesceable(): 

2324 continue 

2325 if vdi in self._failedCoalesceTargets: 

2326 continue 

2327 if vdi.getConfig(vdi.DB_ONBOOT) == vdi.ONBOOT_RESET: 

2328 Util.log("Skipping reset-on-boot %s" % vdi) 

2329 continue 

2330 if vdi.getConfig(vdi.DB_ALLOW_CACHING): 

2331 Util.log("Skipping allow_caching=true %s" % vdi) 

2332 continue 

2333 if vdi.getConfig(vdi.DB_LEAFCLSC) == vdi.LEAFCLSC_DISABLED: 

2334 Util.log("Leaf-coalesce disabled for %s" % vdi) 

2335 continue 

2336 if not (AUTO_ONLINE_LEAF_COALESCE_ENABLED or 

2337 vdi.getConfig(vdi.DB_LEAFCLSC) == vdi.LEAFCLSC_FORCE): 

2338 continue 

2339 candidates.append(vdi) 

2340 

2341 def coalesce(self, vdi, dryRun=False): 

2342 """Coalesce vdi onto parent""" 

2343 Util.log("Coalescing %s -> %s" % (vdi, vdi.parent)) 

2344 if dryRun: 2344 ↛ 2345line 2344 didn't jump to line 2345, because the condition on line 2344 was never true

2345 return 

2346 

2347 try: 

2348 self._coalesce(vdi) 

2349 except util.SMException as e: 

2350 if isinstance(e, AbortException): 2350 ↛ 2351line 2350 didn't jump to line 2351, because the condition on line 2350 was never true

2351 self.cleanup() 

2352 raise 

2353 else: 

2354 self._failedCoalesceTargets.append(vdi) 

2355 Util.logException("coalesce") 

2356 Util.log("Coalesce failed, skipping") 

2357 self.cleanup() 

2358 

2359 def coalesceLeaf(self, vdi, dryRun=False): 

2360 """Leaf-coalesce vdi onto parent""" 

2361 Util.log("Leaf-coalescing %s -> %s" % (vdi, vdi.parent)) 

2362 if dryRun: 

2363 return 

2364 

2365 try: 

2366 uuid = vdi.uuid 

2367 try: 

2368 # "vdi" object will no longer be valid after this call 

2369 if vdi.cowutil.isCoalesceableOnRemote(): 

2370 Util.log("We will live coalesce leaf: {uuid}".format(uuid=vdi.uuid)) 

2371 self._liveLeafCoalesce(vdi, coalesce_on_remote=True) 

2372 else: 

2373 Util.log("We can't live coalesce leaf: {uuid}".format(uuid=vdi.uuid)) 

2374 self._coalesceLeaf(vdi) 

2375 finally: 

2376 vdi = self.getVDI(uuid) 

2377 if vdi: 

2378 vdi.delConfig(vdi.DB_LEAFCLSC) 

2379 except AbortException: 

2380 self.cleanup() 

2381 raise 

2382 except (util.SMException, XenAPI.Failure) as e: 

2383 self._failedCoalesceTargets.append(vdi) 

2384 Util.logException("leaf-coalesce") 

2385 Util.log("Leaf-coalesce failed on %s, skipping" % vdi) 

2386 self.cleanup() 

2387 

2388 def garbageCollect(self, dryRun=False): 

2389 vdiList = self.findGarbage() 

2390 Util.log("Found %d VDIs for deletion:" % len(vdiList)) 

2391 for vdi in vdiList: 

2392 Util.log(" %s" % vdi) 

2393 if not dryRun: 

2394 self.deleteVDIs(vdiList) 

2395 self.cleanupJournals(dryRun) 

2396 

2397 def findGarbage(self): 

2398 vdiList = [] 

2399 for vdi in self.vdiTrees: 

2400 vdiList.extend(vdi.getAllPrunable()) 

2401 return vdiList 

2402 

2403 def deleteVDIs(self, vdiList) -> None: 

2404 for vdi in vdiList: 

2405 if IPCFlag(self.uuid).test(FLAG_TYPE_ABORT): 

2406 raise AbortException("Aborting due to signal") 

2407 Util.log("Deleting unlinked VDI %s" % vdi) 

2408 self.deleteVDI(vdi) 

2409 

2410 def deleteVDI(self, vdi) -> None: 

2411 assert(len(vdi.children) == 0) 

2412 del self.vdis[vdi.uuid] 

2413 if vdi.parent: 2413 ↛ 2415line 2413 didn't jump to line 2415, because the condition on line 2413 was never false

2414 vdi.parent.children.remove(vdi) 

2415 if vdi in self.vdiTrees: 2415 ↛ 2416line 2415 didn't jump to line 2416, because the condition on line 2415 was never true

2416 self.vdiTrees.remove(vdi) 

2417 vdi.delete() 

2418 

2419 def forgetVDI(self, vdiUuid) -> None: 

2420 self.xapi.forgetVDI(self.uuid, vdiUuid) 

2421 

2422 def pauseVDIs(self, vdiList) -> None: 

2423 paused = [] 

2424 failed = False 

2425 for vdi in vdiList: 

2426 try: 

2427 vdi.pause() 

2428 paused.append(vdi) 

2429 except: 

2430 Util.logException("pauseVDIs") 

2431 failed = True 

2432 break 

2433 

2434 if failed: 

2435 self.unpauseVDIs(paused) 

2436 raise util.SMException("Failed to pause VDIs") 

2437 

2438 def unpauseVDIs(self, vdiList): 

2439 failed = False 

2440 for vdi in vdiList: 

2441 try: 

2442 vdi.unpause() 

2443 except: 

2444 Util.log("ERROR: Failed to unpause VDI %s" % vdi) 

2445 failed = True 

2446 if failed: 

2447 raise util.SMException("Failed to unpause VDIs") 

2448 

2449 def getFreeSpace(self) -> int: 

2450 return 0 

2451 

2452 def cleanup(self): 

2453 Util.log("In cleanup") 

2454 return 

2455 

2456 @override 

2457 def __str__(self) -> str: 

2458 if self.name: 

2459 ret = "%s ('%s')" % (self.uuid[0:4], self.name) 

2460 else: 

2461 ret = "%s" % self.uuid 

2462 return ret 

2463 

2464 def lock(self): 

2465 """Acquire the SR lock. Nested acquire()'s are ok. Check for Abort 

2466 signal to avoid deadlocking (trying to acquire the SR lock while the 

2467 lock is held by a process that is trying to abort us)""" 

2468 if not self._srLock: 

2469 return 

2470 

2471 if self._locked == 0: 

2472 abortFlag = IPCFlag(self.uuid) 

2473 for i in range(SR.LOCK_RETRY_ATTEMPTS_LOCK): 

2474 if self._srLock.acquireNoblock(): 

2475 self._locked += 1 

2476 return 

2477 if abortFlag.test(FLAG_TYPE_ABORT): 

2478 raise AbortException("Abort requested") 

2479 time.sleep(SR.LOCK_RETRY_INTERVAL) 

2480 raise util.SMException("Unable to acquire the SR lock") 

2481 

2482 self._locked += 1 

2483 

2484 def unlock(self): 

2485 if not self._srLock: 2485 ↛ 2487line 2485 didn't jump to line 2487, because the condition on line 2485 was never false

2486 return 

2487 assert(self._locked > 0) 

2488 self._locked -= 1 

2489 if self._locked == 0: 

2490 self._srLock.release() 

2491 

2492 def needUpdateBlockInfo(self) -> bool: 

2493 for vdi in self.vdis.values(): 

2494 if vdi.scanError or len(vdi.children) == 0: 

2495 continue 

2496 if not vdi.getConfig(vdi.DB_VDI_BLOCKS): 

2497 return True 

2498 return False 

2499 

2500 def updateBlockInfo(self) -> None: 

2501 for vdi in self.vdis.values(): 

2502 if vdi.scanError or len(vdi.children) == 0: 

2503 continue 

2504 if not vdi.getConfig(vdi.DB_VDI_BLOCKS): 

2505 vdi.updateBlockInfo() 

2506 

2507 def cleanupCoalesceJournals(self): 

2508 """Remove stale coalesce VDI indicators""" 

2509 entries = self.journaler.getAll(VDI.JRN_COALESCE) 

2510 for uuid, jval in entries.items(): 

2511 self.journaler.remove(VDI.JRN_COALESCE, uuid) 

2512 

2513 def cleanupJournals(self, dryRun=False): 

2514 """delete journal entries for non-existing VDIs""" 

2515 for t in [LVMVDI.JRN_ZERO, VDI.JRN_RELINK, SR.JRN_CLONE]: 

2516 entries = self.journaler.getAll(t) 

2517 for uuid, jval in entries.items(): 

2518 if self.getVDI(uuid): 

2519 continue 

2520 if t == SR.JRN_CLONE: 

2521 baseUuid, clonUuid = jval.split("_") 

2522 if self.getVDI(baseUuid): 

2523 continue 

2524 Util.log(" Deleting stale '%s' journal entry for %s " 

2525 "(%s)" % (t, uuid, jval)) 

2526 if not dryRun: 

2527 self.journaler.remove(t, uuid) 

2528 

2529 def cleanupCache(self, maxAge=-1) -> int: 

2530 return 0 

2531 

2532 def hasLeavesAttachedOn(self, vdi: VDI): 

2533 leaves = vdi.getAllLeaves() 

2534 leaves_vdi = [leaf.uuid for leaf in leaves] 

2535 return util.get_hosts_attached_on_with_vdi_uuid(self.xapi.session, leaves_vdi) 

2536 

2537 def _gc_running_file(self, vdi: VDI): 

2538 run_file = "gc_running_{}".format(vdi.uuid) 

2539 return os.path.join(NON_PERSISTENT_DIR, str(self.uuid), run_file) 

2540 

2541 def _create_running_file(self, vdi: VDI): 

2542 with open(self._gc_running_file(vdi), "w") as f: 

2543 f.write("1") 

2544 

2545 def _delete_running_file(self, vdi: VDI): 

2546 os.unlink(self._gc_running_file(vdi)) 

2547 

2548 def _coalesce(self, vdi: VDI): 

2549 list_not_to_relink = None 

2550 if self.journaler.get(vdi.JRN_RELINK, vdi.uuid): 2550 ↛ 2553line 2550 didn't jump to line 2553, because the condition on line 2550 was never true

2551 # this means we had done the actual coalescing already and just 

2552 # need to finish relinking and/or refreshing the children 

2553 Util.log("==> Coalesce apparently already done: skipping") 

2554 

2555 # The parent volume must be active for the parent change to occur. 

2556 # The parent volume may become inactive if the host is rebooted. 

2557 vdi._ensureParentActiveForRelink() 

2558 else: 

2559 # JRN_COALESCE is used to check which VDI is being coalesced in 

2560 # order to decide whether to abort the coalesce. We remove the 

2561 # journal as soon as the COW coalesce step is done, because we 

2562 # don't expect the rest of the process to take long 

2563 

2564 if os.path.exists(self._gc_running_file(vdi)): 2564 ↛ 2565line 2564 didn't jump to line 2565, because the condition on line 2564 was never true

2565 util.SMlog("gc_running already exist for {}. Ignoring...".format(self.uuid)) 

2566 

2567 self._create_running_file(vdi) 

2568 

2569 self.journaler.create(vdi.JRN_COALESCE, vdi.uuid, "1") 

2570 host_refs = self.hasLeavesAttachedOn(vdi) 

2571 # This check of multiple host_refs was done earlier in `isCoalesceable` but 

2572 # we recheck here to be sure another leaf wasn't activated in the meantime. 

2573 if vdi.cowutil.isCoalesceableOnRemote() and len(host_refs) > 1: 2573 ↛ 2574line 2573 didn't jump to line 2574, because the condition on line 2573 was never true

2574 Util.log("Not coalesceable, chain activated more than once") 

2575 Util.log(f"VDI '{vdi.uuid}' has leaves attached on {host_refs}") 

2576 raise Exception("Not coalesceable, chain activated more than once") 

2577 

2578 try: 

2579 if host_refs and vdi.cowutil.isCoalesceableOnRemote(): 2579 ↛ 2581line 2579 didn't jump to line 2581, because the condition on line 2579 was never true

2580 #Leaf opened on another host, we need to call online coalesce 

2581 Util.log("Remote coalesce for {}".format(vdi.path)) 

2582 

2583 leaf_for_coalesce_uuid, host_ref = next(iter(host_refs.items())) # First host_ref since we should only have one 

2584 leaf_for_coalesce = [leaf for leaf in vdi.getAllLeaves() if leaf.uuid == leaf_for_coalesce_uuid][0] 

2585 

2586 vdi._doCoalesceOnHost(host_ref, leaf_for_coalesce) 

2587 # If we use a host OpaqueRef to do a online coalesce, this vdi will not need to be relinked since it was done by tapdisk 

2588 # If we coalesce up the chain, we shouldn't need to do the relink at all, we only need to do the relink on the children if their direct parent was the one we were coalescing 

2589 for child in vdi.children: 

2590 real_parent_uuid = child.extractUuid(child.getParent()) 

2591 if real_parent_uuid == vdi.parent.uuid: 

2592 child._update_vhd_parent(real_parent_uuid) # We update the sm-config:vhd-parent value for this VDI since it has already been relinked 

2593 list_not_to_relink = [leaf.uuid for leaf in child.getAllLeaves()] 

2594 else: 

2595 Util.log("Offline coalesce for {}".format(vdi.path)) 

2596 vdi._doCoalesce() 

2597 except Exception as e: 

2598 Util.log("EXCEPTION while coalescing: {}".format(e)) 

2599 self._delete_running_file(vdi) 

2600 raise 

2601 

2602 self.journaler.remove(vdi.JRN_COALESCE, vdi.uuid) 

2603 self._delete_running_file(vdi) 

2604 

2605 util.fistpoint.activate("LVHDRT_before_create_relink_journal", self.uuid) 

2606 

2607 # we now need to relink the children: lock the SR to prevent ops 

2608 # like SM.clone from manipulating the VDIs we'll be relinking and 

2609 # rescan the SR first in case the children changed since the last 

2610 # scan 

2611 self.journaler.create(vdi.JRN_RELINK, vdi.uuid, "1") 

2612 

2613 self.lock() 

2614 try: 

2615 vdi.parent._tagChildrenForRelink(list_not_to_relink) 

2616 self.scan() 

2617 vdi._relinkSkip() 

2618 finally: 

2619 self.unlock() 

2620 # Reload the children to leave things consistent 

2621 vdi.parent._reloadChildren(vdi) 

2622 self.journaler.remove(vdi.JRN_RELINK, vdi.uuid) 

2623 

2624 self.deleteVDI(vdi) 

2625 

2626 class CoalesceTracker: 

2627 GRACE_ITERATIONS = 2 

2628 MAX_ITERATIONS_NO_PROGRESS = 3 

2629 MAX_ITERATIONS = 20 

2630 MAX_INCREASE_FROM_MINIMUM = 1.2 

2631 HISTORY_STRING = "Iteration: {its} -- Initial size {initSize}" \ 

2632 " --> Final size {finSize}" 

2633 

2634 def __init__(self, sr): 

2635 self.itsNoProgress = 0 

2636 self.its = 0 

2637 self.minSize = float("inf") 

2638 self._history = [] 

2639 self.reason = "" 

2640 self.startSize = None 

2641 self.finishSize = None 

2642 self.sr = sr 

2643 self.grace_remaining = self.GRACE_ITERATIONS 

2644 

2645 @property 

2646 def history(self): 

2647 return [x['msg'] for x in self._history] 

2648 

2649 def moving_average(self): 

2650 """ 

2651 Calculate a three point moving average 

2652 """ 

2653 assert len(self._history) >= 3 

2654 

2655 mv_average = sum([x['finalsize'] for x in self._history]) / len(self._history) 

2656 util.SMlog(f'Calculated moving average as {mv_average}') 

2657 return mv_average 

2658 

2659 def abortCoalesce(self, prevSize, curSize): 

2660 self.its += 1 

2661 self._history.append( 

2662 { 

2663 'finalsize': curSize, 

2664 'msg': self.HISTORY_STRING.format(its=self.its, 

2665 initSize=prevSize, 

2666 finSize=curSize) 

2667 } 

2668 ) 

2669 

2670 self.finishSize = curSize 

2671 

2672 if self.startSize is None: 

2673 self.startSize = prevSize 

2674 

2675 if curSize < self.minSize: 

2676 self.minSize = curSize 

2677 

2678 if prevSize < self.minSize: 

2679 self.minSize = prevSize 

2680 

2681 if self.its < 4: 

2682 # Perform at least three iterations 

2683 return False 

2684 

2685 if prevSize >= curSize or curSize < self.moving_average(): 

2686 # We made progress 

2687 return False 

2688 else: 

2689 self.itsNoProgress += 1 

2690 Util.log("No progress, attempt:" 

2691 " {attempt}".format(attempt=self.itsNoProgress)) 

2692 util.fistpoint.activate("cleanup_tracker_no_progress", self.sr.uuid) 

2693 

2694 if self.its > self.MAX_ITERATIONS: 

2695 max = self.MAX_ITERATIONS 

2696 self.reason = \ 

2697 "Max iterations ({max}) exceeded".format(max=max) 

2698 return True 

2699 

2700 if self.itsNoProgress > self.MAX_ITERATIONS_NO_PROGRESS: 

2701 max = self.MAX_ITERATIONS_NO_PROGRESS 

2702 self.reason = \ 

2703 "No progress made for {max} iterations".format(max=max) 

2704 return True 

2705 

2706 maxSizeFromMin = self.MAX_INCREASE_FROM_MINIMUM * self.minSize 

2707 if curSize > maxSizeFromMin: 

2708 self.grace_remaining -= 1 

2709 if self.grace_remaining == 0: 

2710 self.reason = "Unexpected bump in size," \ 

2711 " compared to minimum achieved" 

2712 

2713 return True 

2714 

2715 return False 

2716 

2717 def printSizes(self): 

2718 Util.log("Starting size was {size}" 

2719 .format(size=self.startSize)) 

2720 Util.log("Final size was {size}" 

2721 .format(size=self.finishSize)) 

2722 Util.log("Minimum size achieved was {size}" 

2723 .format(size=self.minSize)) 

2724 

2725 def printReasoning(self): 

2726 Util.log("Aborted coalesce") 

2727 for hist in self.history: 

2728 Util.log(hist) 

2729 Util.log(self.reason) 

2730 self.printSizes() 

2731 

2732 def printSummary(self): 

2733 if self.its == 0: 

2734 return 

2735 

2736 if self.reason: 2736 ↛ 2737line 2736 didn't jump to line 2737, because the condition on line 2736 was never true

2737 Util.log("Aborted coalesce") 

2738 Util.log(self.reason) 

2739 else: 

2740 Util.log("Coalesce summary") 

2741 

2742 Util.log(f"Performed {self.its} iterations") 

2743 self.printSizes() 

2744 

2745 

2746 def _coalesceLeaf(self, vdi): 

2747 """Leaf-coalesce VDI vdi. Return true if we succeed, false if we cannot 

2748 complete due to external changes, namely vdi_delete and vdi_snapshot 

2749 that alter leaf-coalescibility of vdi""" 

2750 tracker = self.CoalesceTracker(self) 

2751 while not vdi.canLiveCoalesce(self.getStorageSpeed()): 

2752 prevSizePhys = vdi.getSizePhys() 

2753 if not self._snapshotCoalesce(vdi): 2753 ↛ 2754line 2753 didn't jump to line 2754, because the condition on line 2753 was never true

2754 return False 

2755 if tracker.abortCoalesce(prevSizePhys, vdi.getSizePhys()): 

2756 tracker.printReasoning() 

2757 raise util.SMException("VDI {uuid} could not be coalesced" 

2758 .format(uuid=vdi.uuid)) 

2759 tracker.printSummary() 

2760 return self._liveLeafCoalesce(vdi) 

2761 

2762 def calcStorageSpeed(self, startTime, endTime, coalescedSize): 

2763 speed = None 

2764 total_time = endTime - startTime 

2765 if total_time > 0: 

2766 speed = float(coalescedSize) / float(total_time) 

2767 return speed 

2768 

2769 def writeSpeedToFile(self, speed): 

2770 content = [] 

2771 speedFile = None 

2772 path = SPEED_LOG_ROOT.format(uuid=self.uuid) 

2773 self.lock() 

2774 try: 

2775 Util.log("Writing to file: {myfile}".format(myfile=path)) 

2776 lines = "" 

2777 if not os.path.isfile(path): 

2778 lines = str(speed) + "\n" 

2779 else: 

2780 speedFile = open(path, "r+") 

2781 content = speedFile.readlines() 

2782 content.append(str(speed) + "\n") 

2783 if len(content) > N_RUNNING_AVERAGE: 

2784 del content[0] 

2785 lines = "".join(content) 

2786 

2787 util.atomicFileWrite(path, VAR_RUN, lines) 

2788 finally: 

2789 if speedFile is not None: 

2790 speedFile.close() 

2791 Util.log("Closing file: {myfile}".format(myfile=path)) 

2792 self.unlock() 

2793 

2794 def recordStorageSpeed(self, startTime, endTime, coalescedSize): 

2795 speed = self.calcStorageSpeed(startTime, endTime, coalescedSize) 

2796 if speed is None: 

2797 return 

2798 

2799 self.writeSpeedToFile(speed) 

2800 

2801 def getStorageSpeed(self): 

2802 speedFile = None 

2803 path = SPEED_LOG_ROOT.format(uuid=self.uuid) 

2804 self.lock() 

2805 try: 

2806 speed = None 

2807 if os.path.isfile(path): 

2808 speedFile = open(path) 

2809 content = speedFile.readlines() 

2810 try: 

2811 content = [float(i) for i in content] 

2812 except ValueError: 

2813 Util.log("Something bad in the speed log:{log}". 

2814 format(log=speedFile.readlines())) 

2815 return speed 

2816 

2817 if len(content): 

2818 speed = sum(content) / float(len(content)) 

2819 if speed <= 0: 2819 ↛ 2821line 2819 didn't jump to line 2821, because the condition on line 2819 was never true

2820 # Defensive, should be impossible. 

2821 Util.log("Bad speed: {speed} calculated for SR: {uuid}". 

2822 format(speed=speed, uuid=self.uuid)) 

2823 speed = None 

2824 else: 

2825 Util.log("Speed file empty for SR: {uuid}". 

2826 format(uuid=self.uuid)) 

2827 else: 

2828 Util.log("Speed log missing for SR: {uuid}". 

2829 format(uuid=self.uuid)) 

2830 return speed 

2831 finally: 

2832 if not (speedFile is None): 

2833 speedFile.close() 

2834 self.unlock() 

2835 

2836 def _snapshotCoalesce(self, vdi): 

2837 # Note that because we are not holding any locks here, concurrent SM 

2838 # operations may change this tree under our feet. In particular, vdi 

2839 # can be deleted, or it can be snapshotted. 

2840 assert(AUTO_ONLINE_LEAF_COALESCE_ENABLED) 

2841 Util.log("Single-snapshotting %s" % vdi) 

2842 util.fistpoint.activate("LVHDRT_coaleaf_delay_1", self.uuid) 

2843 try: 

2844 ret = self.xapi.singleSnapshotVDI(vdi) 

2845 Util.log("Single-snapshot returned: %s" % ret) 

2846 except XenAPI.Failure as e: 

2847 if util.isInvalidVDI(e): 

2848 Util.log("The VDI appears to have been concurrently deleted") 

2849 return False 

2850 raise 

2851 self.scanLocked() 

2852 tempSnap = vdi.parent 

2853 if not tempSnap.isCoalesceable(): 

2854 Util.log("The VDI appears to have been concurrently snapshotted") 

2855 return False 

2856 Util.log("Coalescing parent %s" % tempSnap) 

2857 util.fistpoint.activate("LVHDRT_coaleaf_delay_2", self.uuid) 

2858 sizePhys = vdi.getSizePhys() 

2859 self._coalesce(tempSnap) 

2860 if not vdi.isLeafCoalesceable(): 

2861 Util.log("The VDI tree appears to have been altered since") 

2862 return False 

2863 return True 

2864 

2865 def _liveLeafCoalesce(self, vdi: VDI, coalesce_on_remote: bool = False) -> bool: 

2866 util.fistpoint.activate("LVHDRT_coaleaf_delay_3", self.uuid) 

2867 self.lock() 

2868 try: 

2869 self.scan() 

2870 if not self.getVDI(vdi.uuid): 

2871 Util.log("The VDI appears to have been deleted meanwhile") 

2872 return False 

2873 if not vdi.isLeafCoalesceable(): 

2874 Util.log("The VDI is no longer leaf-coalesceable") 

2875 return False 

2876 

2877 uuid = vdi.uuid 

2878 if not coalesce_on_remote: 

2879 vdi.pause(failfast=True) 

2880 try: 

2881 try: 

2882 self._create_running_file(vdi) 

2883 # "vdi" object will no longer be valid after this call 

2884 self._doCoalesceLeaf(vdi, coalesce_on_remote) 

2885 except: 

2886 Util.logException("_doCoalesceLeaf") 

2887 self._handleInterruptedCoalesceLeaf() 

2888 raise 

2889 finally: 

2890 vdi = self.getVDI(uuid) 

2891 if vdi: 

2892 vdi.ensureUnpaused() 

2893 self._delete_running_file(vdi) 

2894 vdiOld = self.getVDI(self.TMP_RENAME_PREFIX + uuid) 

2895 if vdiOld: 

2896 util.fistpoint.activate("LVHDRT_coaleaf_before_delete", self.uuid) 

2897 self.deleteVDI(vdiOld) 

2898 util.fistpoint.activate("LVHDRT_coaleaf_after_delete", self.uuid) 

2899 finally: 

2900 self.cleanup() 

2901 self.unlock() 

2902 self.logFilter.logState() 

2903 return True 

2904 

2905 def _doCoalesceLeaf(self, vdi: VDI, coalesce_on_remote: bool): 

2906 """Actual coalescing of a leaf VDI onto parent. Must be called in an 

2907 offline/atomic context""" 

2908 self.journaler.create(VDI.JRN_LEAF, vdi.uuid, vdi.parent.uuid) 

2909 self._prepareCoalesceLeaf(vdi) 

2910 vdi.parent._setHidden(False) 

2911 vdi.parent._increaseSizeVirt(vdi.sizeVirt, False) 

2912 host_refs = self.hasLeavesAttachedOn(vdi) if coalesce_on_remote else None 

2913 if host_refs: 2913 ↛ 2914line 2913 didn't jump to line 2914, because the condition on line 2913 was never true

2914 util.fistpoint.activate("LVHDRT_coaleaf_before_coalesce", self.uuid) 

2915 _, host_ref = next(iter(host_refs.items())) 

2916 vdi._coalesceCowImageOnHost(host_ref, vdi) # vdi is the leaf for the online coalesce 

2917 util.fistpoint.activate("LVHDRT_coaleaf_after_coalesce", self.uuid) 

2918 vdi.pause(failfast=True) 

2919 # We make a pause here after the online coalesce but before the rename so we can refresh the chain for tapdisk.  

2920 # It's also needed to be paused for the rename on slaves with LVMSR. 

2921 # We let the caller `_liveLeafCoalesce` do the unpause with the call to `vdi.ensureUnpaused()` 

2922 else: 

2923 vdi.validate(True) 

2924 vdi.parent.validate(True) 

2925 util.fistpoint.activate("LVHDRT_coaleaf_before_coalesce", self.uuid) 

2926 timeout = vdi.LIVE_LEAF_COALESCE_TIMEOUT 

2927 if vdi.getConfig(vdi.DB_LEAFCLSC) == vdi.LEAFCLSC_FORCE: 2927 ↛ 2928line 2927 didn't jump to line 2928, because the condition on line 2927 was never true

2928 Util.log("Leaf-coalesce forced, will not use timeout") 

2929 timeout = 0 

2930 vdi._coalesceCowImage(timeout) 

2931 util.fistpoint.activate("LVHDRT_coaleaf_after_coalesce", self.uuid) 

2932 vdi.parent.validate(True) 

2933 #vdi._verifyContents(timeout / 2) 

2934 

2935 # rename 

2936 vdiUuid = vdi.uuid 

2937 oldName = vdi.fileName 

2938 origParentUuid = vdi.parent.uuid 

2939 vdi.rename(self.TMP_RENAME_PREFIX + vdiUuid) 

2940 util.fistpoint.activate("LVHDRT_coaleaf_one_renamed", self.uuid) 

2941 vdi.parent.rename(vdiUuid) 

2942 util.fistpoint.activate("LVHDRT_coaleaf_both_renamed", self.uuid) 

2943 self._updateSlavesOnRename(vdi.parent, oldName, origParentUuid) 

2944 

2945 # Note that "vdi.parent" is now the single remaining leaf and "vdi" is 

2946 # garbage 

2947 

2948 # update the VDI record 

2949 if vdi.parent.vdi_type == VdiType.RAW: 2949 ↛ 2950line 2949 didn't jump to line 2950, because the condition on line 2949 was never true

2950 vdi.parent.setConfig(VDI.DB_VDI_TYPE, VdiType.RAW) 

2951 vdi.parent.delConfig(VDI.DB_VDI_BLOCKS) 

2952 util.fistpoint.activate("LVHDRT_coaleaf_after_vdirec", self.uuid) 

2953 

2954 self._updateNode(vdi) 

2955 

2956 # delete the obsolete leaf & inflate the parent (in that order, to 

2957 # minimize free space requirements) 

2958 parent = vdi.parent 

2959 vdi._setHidden(True) 

2960 vdi.parent.children = [] 

2961 vdi.parent = None 

2962 

2963 if parent.parent is None: 

2964 parent.delConfig(VDI.DB_VDI_PARENT) 

2965 

2966 extraSpace = self._calcExtraSpaceNeeded(vdi, parent) 

2967 freeSpace = self.getFreeSpace() 

2968 if freeSpace < extraSpace: 2968 ↛ 2971line 2968 didn't jump to line 2971, because the condition on line 2968 was never true

2969 # don't delete unless we need the space: deletion is time-consuming 

2970 # because it requires contacting the slaves, and we're paused here 

2971 util.fistpoint.activate("LVHDRT_coaleaf_before_delete", self.uuid) 

2972 self.deleteVDI(vdi) 

2973 util.fistpoint.activate("LVHDRT_coaleaf_after_delete", self.uuid) 

2974 

2975 util.fistpoint.activate("LVHDRT_coaleaf_before_remove_j", self.uuid) 

2976 self.journaler.remove(VDI.JRN_LEAF, vdiUuid) 

2977 

2978 self.forgetVDI(origParentUuid) 

2979 self._finishCoalesceLeaf(parent) 

2980 self._updateSlavesOnResize(parent) 

2981 

2982 def _calcExtraSpaceNeeded(self, child, parent) -> int: 

2983 assert(VdiType.isCowImage(parent.vdi_type)) 

2984 extra = child.getSizePhys() - parent.getSizePhys() 

2985 if extra < 0: 2985 ↛ 2986line 2985 didn't jump to line 2986, because the condition on line 2985 was never true

2986 extra = 0 

2987 return extra 

2988 

2989 def _prepareCoalesceLeaf(self, vdi) -> None: 

2990 pass 

2991 

2992 def _updateNode(self, vdi) -> None: 

2993 pass 

2994 

2995 def _finishCoalesceLeaf(self, parent) -> None: 

2996 pass 

2997 

2998 def _updateSlavesOnUndoLeafCoalesce(self, parent, child) -> None: 

2999 pass 

3000 

3001 def _updateSlavesOnRename(self, vdi, oldName, origParentUuid) -> None: 

3002 pass 

3003 

3004 def _updateSlavesOnResize(self, vdi) -> None: 

3005 pass 

3006 

3007 def _removeStaleVDIs(self, uuidsPresent) -> None: 

3008 for uuid in list(self.vdis.keys()): 

3009 if not uuid in uuidsPresent: 

3010 Util.log("VDI %s disappeared since last scan" % \ 

3011 self.vdis[uuid]) 

3012 del self.vdis[uuid] 

3013 

3014 def _handleInterruptedCoalesceLeaf(self) -> None: 

3015 """An interrupted leaf-coalesce operation may leave the COW tree in an 

3016 inconsistent state. If the old-leaf VDI is still present, we revert the 

3017 operation (in case the original error is persistent); otherwise we must 

3018 finish the operation""" 

3019 pass 

3020 

3021 def _buildTree(self, force): 

3022 self.vdiTrees = [] 

3023 for vdi in self.vdis.values(): 

3024 if vdi.parentUuid: 

3025 parent = self.getVDI(vdi.parentUuid) 

3026 if not parent: 

3027 if vdi.uuid.startswith(self.TMP_RENAME_PREFIX): 

3028 self.vdiTrees.append(vdi) 

3029 continue 

3030 if force: 

3031 Util.log("ERROR: Parent VDI %s not found! (for %s)" % \ 

3032 (vdi.parentUuid, vdi.uuid)) 

3033 self.vdiTrees.append(vdi) 

3034 continue 

3035 else: 

3036 raise util.SMException("Parent VDI %s of %s not " \ 

3037 "found" % (vdi.parentUuid, vdi.uuid)) 

3038 vdi.parent = parent 

3039 parent.children.append(vdi) 

3040 else: 

3041 self.vdiTrees.append(vdi) 

3042 

3043 

3044class FileSR(SR): 

3045 TYPE = SR.TYPE_FILE 

3046 CACHE_FILE_EXT = ".vhdcache" 

3047 # cache cleanup actions 

3048 CACHE_ACTION_KEEP = 0 

3049 CACHE_ACTION_REMOVE = 1 

3050 CACHE_ACTION_REMOVE_IF_INACTIVE = 2 

3051 

3052 def __init__(self, uuid, xapi, createLock, force): 

3053 SR.__init__(self, uuid, xapi, createLock, force) 

3054 self.path = "/var/run/sr-mount/%s" % self.uuid 

3055 self.journaler = fjournaler.Journaler(self.path) 

3056 

3057 @override 

3058 def scan(self, force=False) -> None: 

3059 if not util.pathexists(self.path): 

3060 raise util.SMException("directory %s not found!" % self.uuid) 

3061 

3062 uuidsPresent: List[str] = [] 

3063 

3064 for vdi_type in VDI_COW_TYPES: 

3065 scan_result = self._scan(vdi_type, force) 

3066 for uuid, image_info in scan_result.items(): 

3067 vdi = self.getVDI(uuid) 

3068 if not vdi: 

3069 self.logFilter.logNewVDI(uuid) 

3070 vdi = FileVDI(self, uuid, vdi_type) 

3071 self.vdis[uuid] = vdi 

3072 vdi.load(image_info) 

3073 uuidsPresent.extend(scan_result.keys()) 

3074 

3075 rawList = [x for x in os.listdir(self.path) if x.endswith(VdiTypeExtension.RAW)] 

3076 for rawName in rawList: 

3077 uuid = FileVDI.extractUuid(rawName) 

3078 uuidsPresent.append(uuid) 

3079 vdi = self.getVDI(uuid) 

3080 if not vdi: 

3081 self.logFilter.logNewVDI(uuid) 

3082 vdi = FileVDI(self, uuid, VdiType.RAW) 

3083 self.vdis[uuid] = vdi 

3084 self._removeStaleVDIs(uuidsPresent) 

3085 self._buildTree(force) 

3086 self.logFilter.logState() 

3087 self._handleInterruptedCoalesceLeaf() 

3088 

3089 @override 

3090 def getFreeSpace(self) -> int: 

3091 return util.get_fs_size(self.path) - util.get_fs_utilisation(self.path) 

3092 

3093 @override 

3094 def deleteVDIs(self, vdiList) -> None: 

3095 rootDeleted = False 

3096 for vdi in vdiList: 

3097 if not vdi.parent: 

3098 rootDeleted = True 

3099 break 

3100 SR.deleteVDIs(self, vdiList) 

3101 if self.xapi.srRecord["type"] == "nfs" and rootDeleted: 

3102 self.xapi.markCacheSRsDirty() 

3103 

3104 @override 

3105 def cleanupCache(self, maxAge=-1) -> int: 

3106 """Clean up IntelliCache cache files. Caches for leaf nodes are 

3107 removed when the leaf node no longer exists or its allow-caching 

3108 attribute is not set. Caches for parent nodes are removed when the 

3109 parent node no longer exists or it hasn't been used in more than 

3110 <maxAge> hours. 

3111 Return number of caches removed. 

3112 """ 

3113 numRemoved = 0 

3114 cacheFiles = [x for x in os.listdir(self.path) if self._isCacheFileName(x)] 

3115 Util.log("Found %d cache files" % len(cacheFiles)) 

3116 cutoff = datetime.datetime.now() - datetime.timedelta(hours=maxAge) 

3117 for cacheFile in cacheFiles: 

3118 uuid = cacheFile[:-len(self.CACHE_FILE_EXT)] 

3119 action = self.CACHE_ACTION_KEEP 

3120 rec = self.xapi.getRecordVDI(uuid) 

3121 if not rec: 

3122 Util.log("Cache %s: VDI doesn't exist" % uuid) 

3123 action = self.CACHE_ACTION_REMOVE 

3124 elif rec["managed"] and not rec["allow_caching"]: 

3125 Util.log("Cache %s: caching disabled" % uuid) 

3126 action = self.CACHE_ACTION_REMOVE 

3127 elif not rec["managed"] and maxAge >= 0: 

3128 lastAccess = datetime.datetime.fromtimestamp( \ 

3129 os.path.getatime(os.path.join(self.path, cacheFile))) 

3130 if lastAccess < cutoff: 

3131 Util.log("Cache %s: older than %d hrs" % (uuid, maxAge)) 

3132 action = self.CACHE_ACTION_REMOVE_IF_INACTIVE 

3133 

3134 if action == self.CACHE_ACTION_KEEP: 

3135 Util.log("Keeping cache %s" % uuid) 

3136 continue 

3137 

3138 lockId = uuid 

3139 parentUuid = None 

3140 if rec and rec["managed"]: 

3141 parentUuid = rec["sm_config"].get("vhd-parent") 

3142 if parentUuid: 

3143 lockId = parentUuid 

3144 

3145 cacheLock = lock.Lock(blktap2.VDI.LOCK_CACHE_SETUP, lockId) 

3146 cacheLock.acquire() 

3147 try: 

3148 if self._cleanupCache(uuid, action): 

3149 numRemoved += 1 

3150 finally: 

3151 cacheLock.release() 

3152 return numRemoved 

3153 

3154 def _cleanupCache(self, uuid, action): 

3155 assert(action != self.CACHE_ACTION_KEEP) 

3156 rec = self.xapi.getRecordVDI(uuid) 

3157 if rec and rec["allow_caching"]: 

3158 Util.log("Cache %s appears to have become valid" % uuid) 

3159 return False 

3160 

3161 fullPath = os.path.join(self.path, uuid + self.CACHE_FILE_EXT) 

3162 tapdisk = blktap2.Tapdisk.find_by_path(fullPath) 

3163 if tapdisk: 

3164 if action == self.CACHE_ACTION_REMOVE_IF_INACTIVE: 

3165 Util.log("Cache %s still in use" % uuid) 

3166 return False 

3167 Util.log("Shutting down tapdisk for %s" % fullPath) 

3168 tapdisk.shutdown() 

3169 

3170 Util.log("Deleting file %s" % fullPath) 

3171 os.unlink(fullPath) 

3172 return True 

3173 

3174 def _isCacheFileName(self, name): 

3175 return (len(name) == Util.UUID_LEN + len(self.CACHE_FILE_EXT)) and \ 

3176 name.endswith(self.CACHE_FILE_EXT) 

3177 

3178 def _scan(self, vdi_type, force): 

3179 for i in range(SR.SCAN_RETRY_ATTEMPTS): 

3180 error = False 

3181 pattern = os.path.join(self.path, "*%s" % VDI_TYPE_TO_EXTENSION[vdi_type]) 

3182 scan_result = getCowUtil(vdi_type).getAllInfoFromVG(pattern, FileVDI.extractUuid) 

3183 for uuid, vdiInfo in scan_result.items(): 

3184 if vdiInfo.error: 

3185 error = True 

3186 break 

3187 if not error: 

3188 return scan_result 

3189 Util.log("Scan error on attempt %d" % i) 

3190 if force: 

3191 return scan_result 

3192 raise util.SMException("Scan error") 

3193 

3194 @override 

3195 def deleteVDI(self, vdi) -> None: 

3196 self._checkSlaves(vdi) 

3197 SR.deleteVDI(self, vdi) 

3198 

3199 def _checkSlaves(self, vdi): 

3200 onlineHosts = self.xapi.getOnlineHosts() 

3201 abortFlag = IPCFlag(self.uuid) 

3202 for pbdRecord in self.xapi.getAttachedPBDs(): 

3203 hostRef = pbdRecord["host"] 

3204 if hostRef == self.xapi._hostRef: 

3205 continue 

3206 if abortFlag.test(FLAG_TYPE_ABORT): 

3207 raise AbortException("Aborting due to signal") 

3208 try: 

3209 self._checkSlave(hostRef, vdi) 

3210 except XenAPI.Failure: 

3211 if hostRef in onlineHosts: 

3212 raise 

3213 

3214 def _checkSlave(self, hostRef, vdi): 

3215 call = (hostRef, "nfs-on-slave", "check", {'path': vdi.path}) 

3216 Util.log("Checking with slave: %s" % repr(call)) 

3217 _host = self.xapi.session.xenapi.host 

3218 text = _host.call_plugin( * call) 

3219 

3220 @override 

3221 def _handleInterruptedCoalesceLeaf(self) -> None: 

3222 entries = self.journaler.getAll(VDI.JRN_LEAF) 

3223 for uuid, parentUuid in entries.items(): 

3224 fileList = os.listdir(self.path) 

3225 childName = uuid + VdiTypeExtension.VHD 

3226 tmpChildName = self.TMP_RENAME_PREFIX + uuid + VdiTypeExtension.VHD 

3227 parentName1 = parentUuid + VdiTypeExtension.VHD 

3228 parentName2 = parentUuid + VdiTypeExtension.RAW 

3229 parentPresent = (parentName1 in fileList or parentName2 in fileList) 

3230 if parentPresent or tmpChildName in fileList: 

3231 self._undoInterruptedCoalesceLeaf(uuid, parentUuid) 

3232 else: 

3233 self._finishInterruptedCoalesceLeaf(uuid, parentUuid) 

3234 self.journaler.remove(VDI.JRN_LEAF, uuid) 

3235 vdi = self.getVDI(uuid) 

3236 if vdi: 

3237 vdi.ensureUnpaused() 

3238 

3239 def _undoInterruptedCoalesceLeaf(self, childUuid, parentUuid): 

3240 Util.log("*** UNDO LEAF-COALESCE") 

3241 parent = self.getVDI(parentUuid) 

3242 if not parent: 

3243 parent = self.getVDI(childUuid) 

3244 if not parent: 

3245 raise util.SMException("Neither %s nor %s found" % \ 

3246 (parentUuid, childUuid)) 

3247 Util.log("Renaming parent back: %s -> %s" % (childUuid, parentUuid)) 

3248 parent.rename(parentUuid) 

3249 util.fistpoint.activate("LVHDRT_coaleaf_undo_after_rename", self.uuid) 

3250 

3251 child = self.getVDI(childUuid) 

3252 if not child: 

3253 child = self.getVDI(self.TMP_RENAME_PREFIX + childUuid) 

3254 if not child: 

3255 raise util.SMException("Neither %s nor %s found" % \ 

3256 (childUuid, self.TMP_RENAME_PREFIX + childUuid)) 

3257 Util.log("Renaming child back to %s" % childUuid) 

3258 child.rename(childUuid) 

3259 Util.log("Updating the VDI record") 

3260 child.setConfig(VDI.DB_VDI_PARENT, parentUuid) 

3261 child.setConfig(VDI.DB_VDI_TYPE, child.vdi_type) 

3262 util.fistpoint.activate("LVHDRT_coaleaf_undo_after_rename2", self.uuid) 

3263 

3264 if child.isHidden(): 

3265 child._setHidden(False) 

3266 if not parent.isHidden(): 

3267 parent._setHidden(True) 

3268 self._updateSlavesOnUndoLeafCoalesce(parent, child) 

3269 util.fistpoint.activate("LVHDRT_coaleaf_undo_end", self.uuid) 

3270 Util.log("*** leaf-coalesce undo successful") 

3271 if util.fistpoint.is_active("LVHDRT_coaleaf_stop_after_recovery"): 

3272 child.setConfig(VDI.DB_LEAFCLSC, VDI.LEAFCLSC_DISABLED) 

3273 

3274 def _finishInterruptedCoalesceLeaf(self, childUuid, parentUuid): 

3275 Util.log("*** FINISH LEAF-COALESCE") 

3276 vdi = self.getVDI(childUuid) 

3277 if not vdi: 

3278 Util.log(f"_finishInterruptedCoalesceLeaf, vdi {childUuid} not found, aborting") 

3279 raise util.SMException("VDI %s not found" % childUuid) 

3280 try: 

3281 self.forgetVDI(parentUuid) 

3282 except XenAPI.Failure: 

3283 Util.logException('_finishInterruptedCoalesceLeaf') 

3284 pass 

3285 self._updateSlavesOnResize(vdi) 

3286 util.fistpoint.activate("LVHDRT_coaleaf_finish_end", self.uuid) 

3287 Util.log("*** finished leaf-coalesce successfully") 

3288 

3289 

3290class LVMSR(SR): 

3291 TYPE = SR.TYPE_LVHD 

3292 SUBTYPES = ["lvhdoiscsi", "lvhdohba"] 

3293 

3294 def __init__(self, uuid, xapi, createLock, force): 

3295 SR.__init__(self, uuid, xapi, createLock, force) 

3296 self.vgName = "%s%s" % (VG_PREFIX, self.uuid) 

3297 self.path = os.path.join(VG_LOCATION, self.vgName) 

3298 

3299 sr_ref = self.xapi.session.xenapi.SR.get_by_uuid(self.uuid) 

3300 other_conf = self.xapi.session.xenapi.SR.get_other_config(sr_ref) 

3301 lvm_conf = other_conf.get('lvm-conf') if other_conf else None 

3302 self.lvmCache = lvmcache.LVMCache(self.vgName, lvm_conf) 

3303 

3304 self.lvActivator = LVActivator(self.uuid, self.lvmCache) 

3305 self.journaler = journaler.Journaler(self.lvmCache) 

3306 

3307 @override 

3308 def deleteVDI(self, vdi) -> None: 

3309 if self.lvActivator.get(vdi.uuid, False): 

3310 self.lvActivator.deactivate(vdi.uuid, False) 

3311 self._checkSlaves(vdi) 

3312 SR.deleteVDI(self, vdi) 

3313 

3314 @override 

3315 def forgetVDI(self, vdiUuid) -> None: 

3316 SR.forgetVDI(self, vdiUuid) 

3317 mdpath = os.path.join(self.path, lvutil.MDVOLUME_NAME) 

3318 LVMMetadataHandler(mdpath).deleteVdiFromMetadata(vdiUuid) 

3319 

3320 @override 

3321 def getFreeSpace(self) -> int: 

3322 stats = lvutil._getVGstats(self.vgName) 

3323 return stats['physical_size'] - stats['physical_utilisation'] 

3324 

3325 @override 

3326 def cleanup(self): 

3327 if not self.lvActivator.deactivateAll(): 

3328 Util.log("ERROR deactivating LVs while cleaning up") 

3329 

3330 @override 

3331 def needUpdateBlockInfo(self) -> bool: 

3332 for vdi in self.vdis.values(): 

3333 if vdi.scanError or not VdiType.isCowImage(vdi.vdi_type) or len(vdi.children) == 0: 

3334 continue 

3335 if not vdi.getConfig(vdi.DB_VDI_BLOCKS): 

3336 return True 

3337 return False 

3338 

3339 @override 

3340 def updateBlockInfo(self) -> None: 

3341 numUpdated = 0 

3342 for vdi in self.vdis.values(): 

3343 if vdi.scanError or not VdiType.isCowImage(vdi.vdi_type) or len(vdi.children) == 0: 

3344 continue 

3345 if not vdi.getConfig(vdi.DB_VDI_BLOCKS): 

3346 vdi.updateBlockInfo() 

3347 numUpdated += 1 

3348 if numUpdated: 

3349 # deactivate the LVs back sooner rather than later. If we don't 

3350 # now, by the time this thread gets to deactivations, another one 

3351 # might have leaf-coalesced a node and deleted it, making the child 

3352 # inherit the refcount value and preventing the correct decrement 

3353 self.cleanup() 

3354 

3355 @override 

3356 def scan(self, force=False) -> None: 

3357 vdis = self._scan(force) 

3358 for uuid, vdiInfo in vdis.items(): 

3359 vdi = self.getVDI(uuid) 

3360 if not vdi: 

3361 self.logFilter.logNewVDI(uuid) 

3362 vdi = LVMVDI(self, uuid, vdiInfo.vdiType) 

3363 self.vdis[uuid] = vdi 

3364 vdi.load(vdiInfo) 

3365 self._removeStaleVDIs(vdis.keys()) 

3366 self._buildTree(force) 

3367 self.logFilter.logState() 

3368 self._handleInterruptedCoalesceLeaf() 

3369 

3370 def _scan(self, force): 

3371 for i in range(SR.SCAN_RETRY_ATTEMPTS): 

3372 error = False 

3373 self.lvmCache.refresh() 

3374 vdis = LvmCowUtil.getVDIInfo(self.lvmCache) 

3375 for uuid, vdiInfo in vdis.items(): 

3376 if vdiInfo.scanError: 

3377 error = True 

3378 break 

3379 if not error: 

3380 return vdis 

3381 Util.log("Scan error, retrying (%d)" % i) 

3382 if force: 

3383 return vdis 

3384 raise util.SMException("Scan error") 

3385 

3386 @override 

3387 def _removeStaleVDIs(self, uuidsPresent) -> None: 

3388 for uuid in list(self.vdis.keys()): 

3389 if not uuid in uuidsPresent: 

3390 Util.log("VDI %s disappeared since last scan" % \ 

3391 self.vdis[uuid]) 

3392 del self.vdis[uuid] 

3393 if self.lvActivator.get(uuid, False): 

3394 self.lvActivator.remove(uuid, False) 

3395 

3396 @override 

3397 def _liveLeafCoalesce(self, vdi: VDI, coalesce_on_remote: bool = False) -> bool: 

3398 """If the parent is raw and the child was resized (virt. size), then 

3399 we'll need to resize the parent, which can take a while due to zeroing 

3400 out of the extended portion of the LV. Do it before pausing the child 

3401 to avoid a protracted downtime""" 

3402 if not VdiType.isCowImage(vdi.parent.vdi_type) and vdi.sizeVirt > vdi.parent.sizeVirt: 

3403 self.lvmCache.setReadonly(vdi.parent.fileName, False) 

3404 vdi.parent._increaseSizeVirt(vdi.sizeVirt) 

3405 

3406 return SR._liveLeafCoalesce(self, vdi, coalesce_on_remote) 

3407 

3408 @override 

3409 def _prepareCoalesceLeaf(self, vdi) -> None: 

3410 vdi._activateChain() 

3411 self.lvmCache.setReadonly(vdi.parent.fileName, False) 

3412 vdi.deflate() 

3413 vdi.inflateParentForCoalesce() 

3414 

3415 @override 

3416 def _updateNode(self, vdi) -> None: 

3417 # fix the refcounts: the remaining node should inherit the binary 

3418 # refcount from the leaf (because if it was online, it should remain 

3419 # refcounted as such), but the normal refcount from the parent (because 

3420 # this node is really the parent node) - minus 1 if it is online (since 

3421 # non-leaf nodes increment their normal counts when they are online and 

3422 # we are now a leaf, storing that 1 in the binary refcount). 

3423 ns = NS_PREFIX_LVM + self.uuid 

3424 cCnt, cBcnt = RefCounter.check(vdi.uuid, ns) 

3425 pCnt, pBcnt = RefCounter.check(vdi.parent.uuid, ns) 

3426 pCnt = pCnt - cBcnt 

3427 assert(pCnt >= 0) 

3428 RefCounter.set(vdi.parent.uuid, pCnt, cBcnt, ns) 

3429 

3430 @override 

3431 def _finishCoalesceLeaf(self, parent) -> None: 

3432 if not parent.isSnapshot() or parent.isAttachedRW(): 

3433 parent.inflateFully() 

3434 else: 

3435 parent.deflate() 

3436 

3437 @override 

3438 def _calcExtraSpaceNeeded(self, child, parent) -> int: 

3439 return parent.lvmcowutil.calcVolumeSize(parent.sizeVirt) - parent.sizeLV 

3440 

3441 @override 

3442 def _handleInterruptedCoalesceLeaf(self) -> None: 

3443 entries = self.journaler.getAll(VDI.JRN_LEAF) 

3444 for uuid, parentUuid in entries.items(): 

3445 undo = False 

3446 for prefix in LV_PREFIX.values(): 

3447 parentLV = prefix + parentUuid 

3448 undo = self.lvmCache.checkLV(parentLV) 

3449 if undo: 

3450 break 

3451 

3452 if not undo: 

3453 for prefix in LV_PREFIX.values(): 

3454 tmpChildLV = prefix + uuid 

3455 undo = self.lvmCache.checkLV(tmpChildLV) 

3456 if undo: 

3457 break 

3458 

3459 if undo: 

3460 self._undoInterruptedCoalesceLeaf(uuid, parentUuid) 

3461 else: 

3462 self._finishInterruptedCoalesceLeaf(uuid, parentUuid) 

3463 self.journaler.remove(VDI.JRN_LEAF, uuid) 

3464 vdi = self.getVDI(uuid) 

3465 if vdi: 

3466 vdi.ensureUnpaused() 

3467 

3468 def _undoInterruptedCoalesceLeaf(self, childUuid, parentUuid): 

3469 Util.log("*** UNDO LEAF-COALESCE") 

3470 parent = self.getVDI(parentUuid) 

3471 if not parent: 

3472 parent = self.getVDI(childUuid) 

3473 if not parent: 

3474 raise util.SMException("Neither %s nor %s found" % \ 

3475 (parentUuid, childUuid)) 

3476 Util.log("Renaming parent back: %s -> %s" % (childUuid, parentUuid)) 

3477 parent.rename(parentUuid) 

3478 util.fistpoint.activate("LVHDRT_coaleaf_undo_after_rename", self.uuid) 

3479 

3480 child = self.getVDI(childUuid) 

3481 if not child: 

3482 child = self.getVDI(self.TMP_RENAME_PREFIX + childUuid) 

3483 if not child: 

3484 raise util.SMException("Neither %s nor %s found" % \ 

3485 (childUuid, self.TMP_RENAME_PREFIX + childUuid)) 

3486 Util.log("Renaming child back to %s" % childUuid) 

3487 child.rename(childUuid) 

3488 Util.log("Updating the VDI record") 

3489 child.setConfig(VDI.DB_VDI_PARENT, parentUuid) 

3490 child.setConfig(VDI.DB_VDI_TYPE, child.vdi_type) 

3491 util.fistpoint.activate("LVHDRT_coaleaf_undo_after_rename2", self.uuid) 

3492 

3493 # refcount (best effort - assume that it had succeeded if the 

3494 # second rename succeeded; if not, this adjustment will be wrong, 

3495 # leading to a non-deactivation of the LV) 

3496 ns = NS_PREFIX_LVM + self.uuid 

3497 cCnt, cBcnt = RefCounter.check(child.uuid, ns) 

3498 pCnt, pBcnt = RefCounter.check(parent.uuid, ns) 

3499 pCnt = pCnt + cBcnt 

3500 RefCounter.set(parent.uuid, pCnt, 0, ns) 

3501 util.fistpoint.activate("LVHDRT_coaleaf_undo_after_refcount", self.uuid) 

3502 

3503 parent.deflate() 

3504 child.inflateFully() 

3505 util.fistpoint.activate("LVHDRT_coaleaf_undo_after_deflate", self.uuid) 

3506 if child.isHidden(): 

3507 child._setHidden(False) 

3508 if not parent.isHidden(): 

3509 parent._setHidden(True) 

3510 if not parent.lvReadonly: 

3511 self.lvmCache.setReadonly(parent.fileName, True) 

3512 self._updateSlavesOnUndoLeafCoalesce(parent, child) 

3513 util.fistpoint.activate("LVHDRT_coaleaf_undo_end", self.uuid) 

3514 Util.log("*** leaf-coalesce undo successful") 

3515 if util.fistpoint.is_active("LVHDRT_coaleaf_stop_after_recovery"): 

3516 child.setConfig(VDI.DB_LEAFCLSC, VDI.LEAFCLSC_DISABLED) 

3517 

3518 def _finishInterruptedCoalesceLeaf(self, childUuid, parentUuid): 

3519 Util.log("*** FINISH LEAF-COALESCE") 

3520 vdi = self.getVDI(childUuid) 

3521 if not vdi: 

3522 raise util.SMException("VDI %s not found" % childUuid) 

3523 vdi.inflateFully() 

3524 util.fistpoint.activate("LVHDRT_coaleaf_finish_after_inflate", self.uuid) 

3525 try: 

3526 self.forgetVDI(parentUuid) 

3527 except XenAPI.Failure: 

3528 pass 

3529 self._updateSlavesOnResize(vdi) 

3530 util.fistpoint.activate("LVHDRT_coaleaf_finish_end", self.uuid) 

3531 Util.log("*** finished leaf-coalesce successfully") 

3532 

3533 def _checkSlaves(self, vdi): 

3534 """Confirm with all slaves in the pool that 'vdi' is not in use. We 

3535 try to check all slaves, including those that the Agent believes are 

3536 offline, but ignore failures for offline hosts. This is to avoid cases 

3537 where the Agent thinks a host is offline but the host is up.""" 

3538 args = {"vgName": self.vgName, 

3539 "action1": "deactivateNoRefcount", 

3540 "lvName1": vdi.fileName, 

3541 "action2": "cleanupLockAndRefcount", 

3542 "uuid2": vdi.uuid, 

3543 "ns2": NS_PREFIX_LVM + self.uuid} 

3544 onlineHosts = self.xapi.getOnlineHosts() 

3545 abortFlag = IPCFlag(self.uuid) 

3546 for pbdRecord in self.xapi.getAttachedPBDs(): 

3547 hostRef = pbdRecord["host"] 

3548 if hostRef == self.xapi._hostRef: 

3549 continue 

3550 if abortFlag.test(FLAG_TYPE_ABORT): 

3551 raise AbortException("Aborting due to signal") 

3552 Util.log("Checking with slave %s (path %s)" % ( 

3553 self.xapi.getRecordHost(hostRef)['hostname'], vdi.path)) 

3554 try: 

3555 self.xapi.ensureInactive(hostRef, args) 

3556 except XenAPI.Failure: 

3557 if hostRef in onlineHosts: 

3558 raise 

3559 

3560 @override 

3561 def _updateSlavesOnUndoLeafCoalesce(self, parent, child) -> None: 

3562 slaves = util.get_slaves_attached_on(self.xapi.session, [child.uuid]) 

3563 if not slaves: 

3564 Util.log("Update-on-leaf-undo: VDI %s not attached on any slave" % \ 

3565 child) 

3566 return 

3567 

3568 tmpName = child.vdi_type + self.TMP_RENAME_PREFIX + child.uuid 

3569 args = {"vgName": self.vgName, 

3570 "action1": "deactivateNoRefcount", 

3571 "lvName1": tmpName, 

3572 "action2": "deactivateNoRefcount", 

3573 "lvName2": child.fileName, 

3574 "action3": "refresh", 

3575 "lvName3": child.fileName, 

3576 "action4": "refresh", 

3577 "lvName4": parent.fileName} 

3578 for slave in slaves: 

3579 Util.log("Updating %s, %s, %s on slave %s" % \ 

3580 (tmpName, child.fileName, parent.fileName, 

3581 self.xapi.getRecordHost(slave)['hostname'])) 

3582 text = self.xapi.session.xenapi.host.call_plugin( \ 

3583 slave, self.xapi.PLUGIN_ON_SLAVE, "multi", args) 

3584 Util.log("call-plugin returned: '%s'" % text) 

3585 

3586 @override 

3587 def _updateSlavesOnRename(self, vdi, oldNameLV, origParentUuid) -> None: 

3588 slaves = util.get_slaves_attached_on(self.xapi.session, [vdi.uuid]) 

3589 if not slaves: 

3590 Util.log("Update-on-rename: VDI %s not attached on any slave" % vdi) 

3591 return 

3592 

3593 args = {"vgName": self.vgName, 

3594 "action1": "deactivateNoRefcount", 

3595 "lvName1": oldNameLV, 

3596 "action2": "refresh", 

3597 "lvName2": vdi.fileName, 

3598 "action3": "cleanupLockAndRefcount", 

3599 "uuid3": origParentUuid, 

3600 "ns3": NS_PREFIX_LVM + self.uuid} 

3601 for slave in slaves: 

3602 Util.log("Updating %s to %s on slave %s" % \ 

3603 (oldNameLV, vdi.fileName, 

3604 self.xapi.getRecordHost(slave)['hostname'])) 

3605 text = self.xapi.session.xenapi.host.call_plugin( \ 

3606 slave, self.xapi.PLUGIN_ON_SLAVE, "multi", args) 

3607 Util.log("call-plugin returned: '%s'" % text) 

3608 

3609 @override 

3610 def _updateSlavesOnResize(self, vdi) -> None: 

3611 uuids = [x.uuid for x in vdi.getAllLeaves()] 

3612 slaves = util.get_slaves_attached_on(self.xapi.session, uuids) 

3613 if not slaves: 

3614 util.SMlog("Update-on-resize: %s not attached on any slave" % vdi) 

3615 return 

3616 LvmCowUtil.refreshVolumeOnSlaves(self.xapi.session, self.uuid, self.vgName, 

3617 vdi.fileName, vdi.uuid, slaves) 

3618 

3619 

3620class LinstorSR(SR): 

3621 TYPE = SR.TYPE_LINSTOR 

3622 

3623 def __init__(self, uuid, xapi, createLock, force): 

3624 if not LINSTOR_AVAILABLE: 

3625 raise util.SMException( 

3626 'Can\'t load cleanup LinstorSR: LINSTOR libraries are missing' 

3627 ) 

3628 

3629 SR.__init__(self, uuid, xapi, createLock, force) 

3630 self.path = LinstorVolumeManager.DEV_ROOT_PATH 

3631 

3632 class LinstorProxy: 

3633 def __init__(self, sr: LinstorSR) -> None: 

3634 self.sr = sr 

3635 

3636 def __getattr__(self, attr: str) -> Any: 

3637 assert self.sr, "Cannot use `LinstorProxy` without valid `LinstorVolumeManager` instance" 

3638 return getattr(self.sr._linstor, attr) 

3639 

3640 self._linstor = None 

3641 self._linstor_proxy = LinstorProxy(self) 

3642 self._reloadLinstor(journaler_only=True) 

3643 

3644 @override 

3645 def deleteVDI(self, vdi) -> None: 

3646 self._checkSlaves(vdi) 

3647 SR.deleteVDI(self, vdi) 

3648 

3649 @override 

3650 def getFreeSpace(self) -> int: 

3651 return self._linstor.max_volume_size_allowed 

3652 

3653 @override 

3654 def scan(self, force=False) -> None: 

3655 all_vdi_info = self._scan(force) 

3656 for uuid, vdiInfo in all_vdi_info.items(): 

3657 # When vdiInfo is None, the VDI is RAW. 

3658 vdi = self.getVDI(uuid) 

3659 if not vdi: 

3660 self.logFilter.logNewVDI(uuid) 

3661 vdi = LinstorVDI(self, uuid, vdiInfo.vdiType if vdiInfo else VdiType.RAW) 

3662 self.vdis[uuid] = vdi 

3663 if vdiInfo: 

3664 vdi.load(vdiInfo) 

3665 self._removeStaleVDIs(all_vdi_info.keys()) 

3666 self._buildTree(force) 

3667 self.logFilter.logState() 

3668 self._handleInterruptedCoalesceLeaf() 

3669 

3670 @override 

3671 def pauseVDIs(self, vdiList) -> None: 

3672 self._linstor.ensure_volume_list_is_not_locked( 

3673 vdiList, timeout=LinstorVDI.VOLUME_LOCK_TIMEOUT 

3674 ) 

3675 return super(LinstorSR, self).pauseVDIs(vdiList) 

3676 

3677 def _reloadLinstor(self, journaler_only=False): 

3678 session = self.xapi.session 

3679 host_ref = util.get_this_host_ref(session) 

3680 sr_ref = session.xenapi.SR.get_by_uuid(self.uuid) 

3681 

3682 pbd = util.find_my_pbd(session, host_ref, sr_ref) 

3683 if pbd is None: 

3684 raise util.SMException('Failed to find PBD') 

3685 

3686 dconf = session.xenapi.PBD.get_device_config(pbd) 

3687 group_name = dconf['group-name'] 

3688 

3689 controller_uri = get_controller_uri() 

3690 

3691 if not journaler_only: 

3692 self._linstor = LinstorVolumeManager( 

3693 controller_uri, 

3694 group_name, 

3695 repair=True, 

3696 logger=util.SMlog 

3697 ) 

3698 

3699 self.journaler = LinstorJournaler( 

3700 group_name, 

3701 uri=None if self._linstor else controller_uri, 

3702 native_client=self._linstor.native_client if self._linstor else None, 

3703 logger=util.SMlog 

3704 ) 

3705 

3706 def _scan(self, force): 

3707 for i in range(SR.SCAN_RETRY_ATTEMPTS): 

3708 self._reloadLinstor() 

3709 error = False 

3710 try: 

3711 all_vdi_info = self._load_vdi_info() 

3712 for uuid, vdiInfo in all_vdi_info.items(): 

3713 if vdiInfo and vdiInfo.error: 

3714 error = True 

3715 break 

3716 if not error: 

3717 return all_vdi_info 

3718 Util.log('Scan error, retrying ({})'.format(i)) 

3719 except Exception as e: 

3720 Util.log('Scan exception, retrying ({}): {}'.format(i, e)) 

3721 Util.log(traceback.format_exc()) 

3722 

3723 if force: 

3724 return all_vdi_info 

3725 raise util.SMException('Scan error') 

3726 

3727 def _load_vdi_info(self): 

3728 all_volume_info = self._linstor.get_volumes_with_info() 

3729 volumes_metadata = self._linstor.get_volumes_with_metadata() 

3730 

3731 all_vdi_info = {} 

3732 pending_vdis = [] 

3733 

3734 def handle_fail(vdi_uuid, e): 

3735 Util.log(f" [VDI {vdi_uuid}: failed to load VDI info]: {e}") 

3736 info = CowImageInfo(vdi_uuid) 

3737 info.error = 1 

3738 return info 

3739 

3740 for vdi_uuid, volume_info in all_volume_info.items(): 

3741 vdi_type = VdiType.RAW 

3742 try: 

3743 volume_metadata = volumes_metadata[vdi_uuid] 

3744 if not volume_info.name and not list(volume_metadata.items()): 

3745 continue # Ignore it, probably deleted. 

3746 

3747 if vdi_uuid.startswith('DELETED_'): 

3748 # Assume it's really a RAW volume of a failed snap without COW header/footer. 

3749 # We must remove this VDI now without adding it in the VDI list. 

3750 # Otherwise `Relinking` calls and other actions can be launched on it. 

3751 # We don't want that... 

3752 Util.log('Deleting bad VDI {}'.format(vdi_uuid)) 

3753 

3754 self.lock() 

3755 try: 

3756 self._linstor.destroy_volume(vdi_uuid) 

3757 try: 

3758 self.forgetVDI(vdi_uuid) 

3759 except: 

3760 pass 

3761 except Exception as e: 

3762 Util.log('Cannot delete bad VDI: {}'.format(e)) 

3763 finally: 

3764 self.unlock() 

3765 continue 

3766 

3767 vdi_type = volume_metadata.get(VDI_TYPE_TAG) 

3768 if VdiType.isCowImage(vdi_type): 

3769 pending_vdis.append((vdi_uuid, vdi_type)) 

3770 else: 

3771 all_vdi_info[vdi_uuid] = None 

3772 except Exception as e: 

3773 all_vdi_info[vdi_uuid] = handle_fail(vdi_uuid, e) 

3774 

3775 multi_cowutil = MultiLinstorCowUtil(self._linstor.uri, self._linstor.group_name) 

3776 

3777 def load_info(vdi, multi_cowutil): 

3778 vdi_uuid, vdi_type = vdi 

3779 try: 

3780 vdiInfo = multi_cowutil.get_local_cowutil(vdi_type).get_info(vdi_uuid) 

3781 except Exception as e: 

3782 vdiInfo = handle_fail(vdi_uuid, e) 

3783 vdiInfo.vdiType = vdi_type 

3784 return vdiInfo 

3785 

3786 try: 

3787 for vdiInfo in multi_cowutil.run(load_info, pending_vdis): 

3788 all_vdi_info[vdiInfo.uuid] = vdiInfo 

3789 finally: 

3790 del multi_cowutil 

3791 

3792 return all_vdi_info 

3793 

3794 @override 

3795 def _prepareCoalesceLeaf(self, vdi) -> None: 

3796 vdi._activateChain() 

3797 vdi.deflate() 

3798 vdi._inflateParentForCoalesce() 

3799 

3800 @override 

3801 def _finishCoalesceLeaf(self, parent) -> None: 

3802 if not parent.isSnapshot() or parent.isAttachedRW(): 

3803 parent.inflateFully() 

3804 else: 

3805 parent.deflate() 

3806 

3807 @override 

3808 def _calcExtraSpaceNeeded(self, child, parent) -> int: 

3809 return LinstorCowUtil( 

3810 self.xapi.session, self._linstor, parent.vdi_type 

3811 ).compute_volume_size(parent.sizeVirt) - parent.getDrbdSize() 

3812 

3813 def _hasValidDevicePath(self, uuid): 

3814 try: 

3815 self._linstor.get_device_path(uuid) 

3816 except Exception: 

3817 # TODO: Maybe log exception. 

3818 return False 

3819 return True 

3820 

3821 @override 

3822 def _liveLeafCoalesce(self, vdi: VDI, coalesce_on_remote: bool = False) -> bool: 

3823 self.lock() 

3824 try: 

3825 self._linstor.ensure_volume_is_not_locked( 

3826 vdi.uuid, timeout=LinstorVDI.VOLUME_LOCK_TIMEOUT 

3827 ) 

3828 return super(LinstorSR, self)._liveLeafCoalesce(vdi) 

3829 finally: 

3830 self.unlock() 

3831 

3832 @override 

3833 def _handleInterruptedCoalesceLeaf(self) -> None: 

3834 entries = self.journaler.get_all(VDI.JRN_LEAF) 

3835 for uuid, parentUuid in entries.items(): 

3836 if self._hasValidDevicePath(parentUuid) or \ 

3837 self._hasValidDevicePath(self.TMP_RENAME_PREFIX + uuid): 

3838 self._undoInterruptedCoalesceLeaf(uuid, parentUuid) 

3839 else: 

3840 self._finishInterruptedCoalesceLeaf(uuid, parentUuid) 

3841 self.journaler.remove(VDI.JRN_LEAF, uuid) 

3842 vdi = self.getVDI(uuid) 

3843 if vdi: 

3844 vdi.ensureUnpaused() 

3845 

3846 def _undoInterruptedCoalesceLeaf(self, childUuid, parentUuid): 

3847 Util.log('*** UNDO LEAF-COALESCE') 

3848 parent = self.getVDI(parentUuid) 

3849 if not parent: 

3850 parent = self.getVDI(childUuid) 

3851 if not parent: 

3852 raise util.SMException( 

3853 'Neither {} nor {} found'.format(parentUuid, childUuid) 

3854 ) 

3855 Util.log( 

3856 'Renaming parent back: {} -> {}'.format(childUuid, parentUuid) 

3857 ) 

3858 parent.rename(parentUuid) 

3859 

3860 child = self.getVDI(childUuid) 

3861 if not child: 

3862 child = self.getVDI(self.TMP_RENAME_PREFIX + childUuid) 

3863 if not child: 

3864 raise util.SMException( 

3865 'Neither {} nor {} found'.format( 

3866 childUuid, self.TMP_RENAME_PREFIX + childUuid 

3867 ) 

3868 ) 

3869 Util.log('Renaming child back to {}'.format(childUuid)) 

3870 child.rename(childUuid) 

3871 Util.log('Updating the VDI record') 

3872 child.setConfig(VDI.DB_VDI_PARENT, parentUuid) 

3873 child.setConfig(VDI.DB_VDI_TYPE, child.vdi_type) 

3874 

3875 # TODO: Maybe deflate here. 

3876 

3877 if child.isHidden(): 

3878 child._setHidden(False) 

3879 if not parent.isHidden(): 

3880 parent._setHidden(True) 

3881 self._updateSlavesOnUndoLeafCoalesce(parent, child) 

3882 Util.log('*** leaf-coalesce undo successful') 

3883 

3884 def _finishInterruptedCoalesceLeaf(self, childUuid, parentUuid): 

3885 Util.log('*** FINISH LEAF-COALESCE') 

3886 vdi = self.getVDI(childUuid) 

3887 if not vdi: 

3888 raise util.SMException('VDI {} not found'.format(childUuid)) 

3889 # TODO: Maybe inflate. 

3890 try: 

3891 self.forgetVDI(parentUuid) 

3892 except XenAPI.Failure: 

3893 pass 

3894 self._updateSlavesOnResize(vdi) 

3895 Util.log('*** finished leaf-coalesce successfully') 

3896 

3897 def _checkSlaves(self, vdi): 

3898 try: 

3899 openers = self._linstor.get_volume_openers(vdi.uuid) 

3900 for host_openers in openers.values(): 

3901 for opener in host_openers.values(): 

3902 if opener['process-name'] != 'tapdisk': 

3903 raise util.SMException( 

3904 'VDI {} is in use: {}'.format(vdi.uuid, openers) 

3905 ) 

3906 except LinstorVolumeManagerError as e: 

3907 if e.code != LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS: 

3908 raise 

3909 

3910 @classmethod 

3911 def abort_gc_from_openers_vdi(cls, vdi_uuid: str, openers: "LinstorVolumeOpeners") -> bool: 

3912 return cls._abort_gc_from_openers(vdi_uuid, True, openers) 

3913 

3914 @classmethod 

3915 def abort_gc_from_openers_sr(cls, sr_uuid: str, openers: "LinstorVolumeOpeners") -> bool: 

3916 return cls._abort_gc_from_openers(sr_uuid, False, openers) 

3917 

3918 @staticmethod 

3919 def _abort_gc_from_openers(uuid: str, is_vdi_uuid: bool, openers: "LinstorVolumeOpeners") -> bool: 

3920 from linstorcowutil import MANAGER_PLUGIN 

3921 

3922 node_name = None 

3923 

3924 for host_openers in openers.values(): 

3925 for hostname, opener in host_openers.items(): 

3926 # Not the most accurate check but it works... 

3927 # `vhd-util` is probably prefixed with a "+" which is ignored here. 

3928 if not opener["process-name"].endswith("vhd-util") or "coalesce" not in opener["cmdline"]: 

3929 continue 

3930 

3931 if not node_name: 

3932 import socket 

3933 node_name = socket.gethostname() 

3934 

3935 if node_name == hostname: 

3936 continue 

3937 

3938 with util.timeout(5): 

3939 session = XAPI.getSession() 

3940 try: 

3941 sr_uuid = util.get_sr_uuid_from_vdi_uuid(session, uuid) if is_vdi_uuid else uuid 

3942 util.SMlog(f"LINSTOR volume is coalescing on `{sr_uuid}`. We're going to interrupt the GC...") 

3943 return util.strtobool(session.xenapi.host.call_plugin( 

3944 util.get_master_ref(session), MANAGER_PLUGIN, "abortGc", {"srUuid": sr_uuid} 

3945 )) 

3946 finally: 

3947 session.xenapi.session.logout() 

3948 return False 

3949 

3950 

3951 

3952################################################################################ 

3953# 

3954# Helpers 

3955# 

3956def daemonize(): 

3957 pid = os.fork() 

3958 if pid: 

3959 os.waitpid(pid, 0) 

3960 Util.log("New PID [%d]" % pid) 

3961 return False 

3962 os.chdir("/") 

3963 os.setsid() 

3964 pid = os.fork() 

3965 if pid: 

3966 Util.log("Will finish as PID [%d]" % pid) 

3967 os._exit(0) 

3968 for fd in [0, 1, 2]: 

3969 try: 

3970 os.close(fd) 

3971 except OSError: 

3972 pass 

3973 # we need to fill those special fd numbers or pread won't work 

3974 sys.stdin = open("/dev/null", 'r') 

3975 sys.stderr = open("/dev/null", 'w') 

3976 sys.stdout = open("/dev/null", 'w') 

3977 # As we're a new process we need to clear the lock objects 

3978 lock.Lock.clearAll() 

3979 return True 

3980 

3981 

3982def normalizeType(type): 

3983 if type in LVMSR.SUBTYPES: 

3984 type = SR.TYPE_LVHD 

3985 if type in ["lvm", "lvmoiscsi", "lvmohba", "lvmofcoe"]: 

3986 # temporary while LVHD is symlinked as LVM 

3987 type = SR.TYPE_LVHD 

3988 if type in [ 

3989 "ext", "nfs", "ocfsoiscsi", "ocfsohba", "smb", "cephfs", "glusterfs", 

3990 "moosefs", "xfs", "zfs", "largeblock" 

3991 ]: 

3992 type = SR.TYPE_FILE 

3993 if type in ["linstor"]: 

3994 type = SR.TYPE_LINSTOR 

3995 if type not in SR.TYPES: 

3996 raise util.SMException("Unsupported SR type: %s" % type) 

3997 return type 

3998 

3999GCPAUSE_DEFAULT_SLEEP = 5 * 60 

4000 

4001 

4002def _gc_init_file(sr_uuid): 

4003 return os.path.join(NON_PERSISTENT_DIR, str(sr_uuid), 'gc_init') 

4004 

4005 

4006def _create_init_file(sr_uuid): 

4007 util.makedirs(os.path.join(NON_PERSISTENT_DIR, str(sr_uuid))) 

4008 with open(os.path.join(_gc_init_file(sr_uuid)), 'w+') as f: 

4009 f.write('1') 

4010 

4011 

4012def _gcLoopPause(sr, dryRun=False, immediate=False): 

4013 if immediate: 

4014 return 

4015 

4016 # Check to see if the GCPAUSE_FISTPOINT is present. If so the fist 

4017 # point will just return. Otherwise, fall back on an abortable sleep. 

4018 

4019 if util.fistpoint.is_active(util.GCPAUSE_FISTPOINT): 

4020 

4021 util.fistpoint.activate_custom_fn(util.GCPAUSE_FISTPOINT, 4021 ↛ exitline 4021 didn't jump to the function exit

4022 lambda *args: None) 

4023 elif os.path.exists(_gc_init_file(sr.uuid)): 

4024 def abortTest(): 

4025 return IPCFlag(sr.uuid).test(FLAG_TYPE_ABORT) 

4026 

4027 # If time.sleep hangs we are in deep trouble, however for 

4028 # completeness we set the timeout of the abort thread to 

4029 # 110% of GCPAUSE_DEFAULT_SLEEP. 

4030 Util.log("GC active, about to go quiet") 

4031 Util.runAbortable(lambda: time.sleep(GCPAUSE_DEFAULT_SLEEP), 4031 ↛ exitline 4031 didn't run the lambda on line 4031

4032 None, sr.uuid, abortTest, VDI.POLL_INTERVAL, 

4033 GCPAUSE_DEFAULT_SLEEP * 1.1) 

4034 Util.log("GC active, quiet period ended") 

4035 

4036 

4037def _gcLoop(sr, dryRun=False, immediate=False): 

4038 if not lockGCActive.acquireNoblock(): 4038 ↛ 4039line 4038 didn't jump to line 4039, because the condition on line 4038 was never true

4039 Util.log("Another GC instance already active, exiting") 

4040 return 

4041 

4042 # Check we're still attached after acquiring locks 

4043 if not sr.xapi.isPluggedHere(): 

4044 Util.log("SR no longer attached, exiting") 

4045 return 

4046 

4047 # Clean up Intellicache files 

4048 sr.cleanupCache() 

4049 

4050 # Track how many we do 

4051 coalesced = 0 

4052 task_status = "success" 

4053 try: 

4054 # Check if any work needs to be done 

4055 if not sr.xapi.isPluggedHere(): 4055 ↛ 4056line 4055 didn't jump to line 4056, because the condition on line 4055 was never true

4056 Util.log("SR no longer attached, exiting") 

4057 return 

4058 sr.scanLocked() 

4059 if not sr.hasWork(): 

4060 Util.log("No work, exiting") 

4061 return 

4062 sr.xapi.create_task( 

4063 "Garbage Collection", 

4064 "Garbage collection for SR %s" % sr.uuid) 

4065 _gcLoopPause(sr, dryRun, immediate=immediate) 

4066 while True: 

4067 if SIGTERM: 

4068 Util.log("Term requested") 

4069 return 

4070 

4071 if not sr.xapi.isPluggedHere(): 4071 ↛ 4072line 4071 didn't jump to line 4072, because the condition on line 4071 was never true

4072 Util.log("SR no longer attached, exiting") 

4073 break 

4074 sr.scanLocked() 

4075 if not sr.hasWork(): 

4076 Util.log("No work, exiting") 

4077 break 

4078 

4079 if not lockGCRunning.acquireNoblock(): 4079 ↛ 4080line 4079 didn't jump to line 4080, because the condition on line 4079 was never true

4080 Util.log("Unable to acquire GC running lock.") 

4081 return 

4082 try: 

4083 if not sr.gcEnabled(): 4083 ↛ 4084line 4083 didn't jump to line 4084, because the condition on line 4083 was never true

4084 break 

4085 

4086 sr.xapi.update_task_progress("done", coalesced) 

4087 

4088 sr.cleanupCoalesceJournals() 

4089 # Create the init file here in case startup is waiting on it 

4090 _create_init_file(sr.uuid) 

4091 sr.scanLocked() 

4092 sr.updateBlockInfo() 

4093 

4094 howmany = len(sr.findGarbage()) 

4095 if howmany > 0: 

4096 Util.log("Found %d orphaned vdis" % howmany) 

4097 sr.lock() 

4098 try: 

4099 sr.garbageCollect(dryRun) 

4100 finally: 

4101 sr.unlock() 

4102 sr.xapi.srUpdate() 

4103 

4104 candidate = sr.findCoalesceable() 

4105 if candidate: 

4106 util.fistpoint.activate( 

4107 "LVHDRT_finding_a_suitable_pair", sr.uuid) 

4108 sr.coalesce(candidate, dryRun) 

4109 sr.xapi.srUpdate() 

4110 coalesced += 1 

4111 continue 

4112 

4113 candidate = sr.findLeafCoalesceable() 

4114 if candidate: 4114 ↛ 4121line 4114 didn't jump to line 4121, because the condition on line 4114 was never false

4115 sr.coalesceLeaf(candidate, dryRun) 

4116 sr.xapi.srUpdate() 

4117 coalesced += 1 

4118 continue 

4119 

4120 finally: 

4121 lockGCRunning.release() 4121 ↛ 4126line 4121 didn't jump to line 4126, because the break on line 4084 wasn't executed

4122 except: 

4123 task_status = "failure" 

4124 raise 

4125 finally: 

4126 sr.xapi.set_task_status(task_status) 

4127 Util.log("GC process exiting, no work left") 

4128 _create_init_file(sr.uuid) 

4129 lockGCActive.release() 

4130 

4131 

4132def _gc(session, srUuid, dryRun=False, immediate=False): 

4133 init(srUuid) 

4134 sr = SR.getInstance(srUuid, session) 

4135 if not sr.gcEnabled(False): 4135 ↛ 4136line 4135 didn't jump to line 4136, because the condition on line 4135 was never true

4136 return 

4137 

4138 try: 

4139 _gcLoop(sr, dryRun, immediate=immediate) 

4140 finally: 

4141 sr.check_no_space_candidates() 

4142 sr.cleanup() 

4143 sr.logFilter.logState() 

4144 del sr.xapi 

4145 

4146 

4147def _abort(srUuid, soft=False): 

4148 """Aborts an GC/coalesce. 

4149 

4150 srUuid: the UUID of the SR whose GC/coalesce must be aborted 

4151 soft: If set to True and there is a pending abort signal, the function 

4152 doesn't do anything. If set to False, a new abort signal is issued. 

4153 

4154 returns: If soft is set to False, we return True holding lockGCActive. If 

4155 soft is set to False and an abort signal is pending, we return False 

4156 without holding lockGCActive. An exception is raised in case of error.""" 

4157 Util.log("=== SR %s: abort ===" % (srUuid)) 

4158 init(srUuid) 

4159 if not lockGCActive.acquireNoblock(): 

4160 gotLock = False 

4161 Util.log("Aborting currently-running instance (SR %s)" % srUuid) 

4162 abortFlag = IPCFlag(srUuid) 

4163 if not abortFlag.set(FLAG_TYPE_ABORT, soft): 

4164 return False 

4165 for i in range(SR.LOCK_RETRY_ATTEMPTS): 

4166 gotLock = lockGCActive.acquireNoblock() 

4167 if gotLock: 

4168 break 

4169 time.sleep(SR.LOCK_RETRY_INTERVAL) 

4170 abortFlag.clear(FLAG_TYPE_ABORT) 

4171 if not gotLock: 

4172 raise util.CommandException(code=errno.ETIMEDOUT, 

4173 reason="SR %s: error aborting existing process" % srUuid) 

4174 return True 

4175 

4176 

4177def init(srUuid): 

4178 global lockGCRunning 

4179 if not lockGCRunning: 4179 ↛ 4180line 4179 didn't jump to line 4180, because the condition on line 4179 was never true

4180 lockGCRunning = lock.Lock(lock.LOCK_TYPE_GC_RUNNING, srUuid) 

4181 global lockGCActive 

4182 if not lockGCActive: 4182 ↛ 4183line 4182 didn't jump to line 4183, because the condition on line 4182 was never true

4183 lockGCActive = LockActive(srUuid) 

4184 

4185 

4186class LockActive: 

4187 """ 

4188 Wraps the use of LOCK_TYPE_GC_ACTIVE such that the lock cannot be acquired 

4189 if another process holds the SR lock. 

4190 """ 

4191 def __init__(self, srUuid): 

4192 self._lock = lock.Lock(LOCK_TYPE_GC_ACTIVE, srUuid) 

4193 self._srLock = lock.Lock(lock.LOCK_TYPE_SR, srUuid) 

4194 

4195 def acquireNoblock(self): 

4196 self._srLock.acquire() 

4197 

4198 try: 

4199 return self._lock.acquireNoblock() 

4200 finally: 

4201 self._srLock.release() 

4202 

4203 def release(self): 

4204 self._lock.release() 

4205 

4206 

4207def usage(): 

4208 output = """Garbage collect and/or coalesce COW images in a COW-based SR 

4209 

4210Parameters: 

4211 -u --uuid UUID SR UUID 

4212 and one of: 

4213 -g --gc garbage collect, coalesce, and repeat while there is work 

4214 -G --gc_force garbage collect once, aborting any current operations 

4215 -c --cache-clean <max_age> clean up IntelliCache cache files older than 

4216 max_age hours 

4217 -a --abort abort any currently running operation (GC or coalesce) 

4218 -q --query query the current state (GC'ing, coalescing or not running) 

4219 -x --disable disable GC/coalesce (will be in effect until you exit) 

4220 -t --debug see Debug below 

4221 

4222Options: 

4223 -b --background run in background (return immediately) (valid for -g only) 

4224 -f --force continue in the presence of COW images with errors (when doing 

4225 GC, this might cause removal of any such images) (only valid 

4226 for -G) (DANGEROUS) 

4227 

4228Debug: 

4229 The --debug parameter enables manipulation of LVHD VDIs for debugging 

4230 purposes. ** NEVER USE IT ON A LIVE VM ** 

4231 The following parameters are required: 

4232 -t --debug <cmd> <cmd> is one of "activate", "deactivate", "inflate", 

4233 "deflate". 

4234 -v --vdi_uuid VDI UUID 

4235 """ 

4236 #-d --dry-run don't actually perform any SR-modifying operations 

4237 print(output) 

4238 Util.log("(Invalid usage)") 

4239 sys.exit(1) 

4240 

4241 

4242############################################################################## 

4243# 

4244# API 

4245# 

4246def abort(srUuid, soft=False): 

4247 """Abort GC/coalesce if we are currently GC'ing or coalescing a VDI pair. 

4248 """ 

4249 if _abort(srUuid, soft): 

4250 stop_gc_service(srUuid) 

4251 Util.log("abort: releasing the process lock") 

4252 lockGCActive.release() 

4253 return True 

4254 else: 

4255 return False 

4256 

4257 

4258def run_gc(session, srUuid, dryRun, immediate=False): 

4259 try: 

4260 _gc(session, srUuid, dryRun, immediate=immediate) 

4261 return 0 

4262 except AbortException: 

4263 Util.log("Aborted") 

4264 return 2 

4265 except Exception: 

4266 Util.logException("gc") 

4267 Util.log("* * * * * SR %s: ERROR\n" % srUuid) 

4268 return 1 

4269 

4270 

4271def gc(session, srUuid, inBackground, dryRun=False): 

4272 """Garbage collect all deleted VDIs in SR "srUuid". Fork & return 

4273 immediately if inBackground=True. 

4274 

4275 The following algorithm is used: 

4276 1. If we are already GC'ing in this SR, return 

4277 2. If we are already coalescing a VDI pair: 

4278 a. Scan the SR and determine if the VDI pair is GC'able 

4279 b. If the pair is not GC'able, return 

4280 c. If the pair is GC'able, abort coalesce 

4281 3. Scan the SR 

4282 4. If there is nothing to collect, nor to coalesce, return 

4283 5. If there is something to collect, GC all, then goto 3 

4284 6. If there is something to coalesce, coalesce one pair, then goto 3 

4285 """ 

4286 Util.log("=== SR %s: gc ===" % srUuid) 

4287 

4288 signal.signal(signal.SIGTERM, receiveSignal) 

4289 

4290 if inBackground: 

4291 if daemonize(): 4291 ↛ exitline 4291 didn't return from function 'gc', because the condition on line 4291 was never false

4292 # we are now running in the background. Catch & log any errors 

4293 # because there is no other way to propagate them back at this 

4294 # point 

4295 

4296 run_gc(None, srUuid, dryRun) 

4297 os._exit(0) 

4298 else: 

4299 os._exit(run_gc(session, srUuid, dryRun, immediate=True)) 

4300 

4301 

4302def start_gc(session, sr_uuid): 

4303 """ 

4304 This function is used to try to start a backgrounded GC session by forking 

4305 the current process. If using the systemd version, call start_gc_service() instead. 

4306 """ 

4307 # don't bother if an instance already running (this is just an 

4308 # optimization to reduce the overhead of forking a new process if we 

4309 # don't have to, but the process will check the lock anyways) 

4310 lockRunning = lock.Lock(lock.LOCK_TYPE_GC_RUNNING, sr_uuid) 

4311 if not lockRunning.acquireNoblock(): 

4312 if should_preempt(session, sr_uuid): 

4313 util.SMlog("Aborting currently-running coalesce of garbage VDI") 

4314 try: 

4315 if not abort(sr_uuid, soft=True): 

4316 util.SMlog("The GC has already been scheduled to re-start") 

4317 except util.CommandException as e: 

4318 if e.code != errno.ETIMEDOUT: 

4319 raise 

4320 util.SMlog('failed to abort the GC') 

4321 else: 

4322 util.SMlog("A GC instance already running, not kicking") 

4323 return 

4324 else: 

4325 lockRunning.release() 

4326 

4327 util.SMlog(f"Starting GC file is {__file__}") 

4328 subprocess.run([__file__, '-b', '-u', sr_uuid, '-g'], 

4329 stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) 

4330 

4331def _gc_service_cmd(sr_uuid, action, extra_args=None): 

4332 """ 

4333 Build and run the systemctl command for the GC service using util.doexec. 

4334 """ 

4335 sr_uuid_esc = sr_uuid.replace("-", "\\x2d") 

4336 cmd=["/usr/bin/systemctl", "--quiet"] 

4337 if extra_args: 

4338 cmd.extend(extra_args) 

4339 cmd += [action, f"SMGC@{sr_uuid_esc}"] 

4340 return util.doexec(cmd) 

4341 

4342 

4343def start_gc_service(sr_uuid, wait=False): 

4344 """ 

4345 This starts the templated systemd service which runs GC on the given SR UUID. 

4346 If the service was already started, this is a no-op. 

4347 

4348 Because the service is a one-shot with RemainAfterExit=no, when called with 

4349 wait=True this will run the service synchronously and will not return until the 

4350 run has finished. This is used to force a run of the GC instead of just kicking it 

4351 in the background. 

4352 """ 

4353 util.SMlog(f"Kicking SMGC@{sr_uuid}...") 

4354 _gc_service_cmd(sr_uuid, "start", extra_args=None if wait else ["--no-block"]) 

4355 

4356 

4357def stop_gc_service(sr_uuid): 

4358 """ 

4359 Stops the templated systemd service which runs GC on the given SR UUID. 

4360 """ 

4361 util.SMlog(f"Stopping SMGC@{sr_uuid}...") 

4362 (rc, _stdout, stderr) = _gc_service_cmd(sr_uuid, "stop") 

4363 if rc != 0: 4363 ↛ exitline 4363 didn't return from function 'stop_gc_service', because the condition on line 4363 was never false

4364 util.SMlog(f"Failed to stop gc service `SMGC@{sr_uuid}`: `{stderr}`") 

4365 

4366 

4367def wait_for_completion(sr_uuid): 

4368 while get_state(sr_uuid): 

4369 time.sleep(5) 

4370 

4371 

4372def gc_force(session, srUuid, force=False, dryRun=False, lockSR=False): 

4373 """Garbage collect all deleted VDIs in SR "srUuid". The caller must ensure 

4374 the SR lock is held. 

4375 The following algorithm is used: 

4376 1. If we are already GC'ing or coalescing a VDI pair, abort GC/coalesce 

4377 2. Scan the SR 

4378 3. GC 

4379 4. return 

4380 """ 

4381 Util.log("=== SR %s: gc_force ===" % srUuid) 

4382 init(srUuid) 

4383 sr = SR.getInstance(srUuid, session, lockSR, True) 

4384 if not lockGCActive.acquireNoblock(): 

4385 abort(srUuid) 

4386 else: 

4387 Util.log("Nothing was running, clear to proceed") 

4388 

4389 if force: 

4390 Util.log("FORCED: will continue even if there are COW image errors") 

4391 sr.scanLocked(force) 

4392 sr.cleanupCoalesceJournals() 

4393 

4394 try: 

4395 sr.cleanupCache() 

4396 sr.garbageCollect(dryRun) 

4397 finally: 

4398 sr.cleanup() 

4399 sr.logFilter.logState() 

4400 lockGCActive.release() 

4401 

4402 

4403def get_state(srUuid): 

4404 """Return whether GC/coalesce is currently running or not. This asks systemd for 

4405 the state of the templated SMGC service and will return True if it is "activating" 

4406 or "running" (for completeness, as in practice it will never achieve the latter state) 

4407 """ 

4408 sr_uuid_esc = srUuid.replace("-", "\\x2d") 

4409 cmd=[ "/usr/bin/systemctl", "is-active", f"SMGC@{sr_uuid_esc}"] 

4410 result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) 

4411 state = result.stdout.decode('utf-8').rstrip() 

4412 if state == "activating" or state == "running": 

4413 return True 

4414 return False 

4415 

4416 

4417def should_preempt(session, srUuid): 

4418 sr = SR.getInstance(srUuid, session) 

4419 entries = sr.journaler.getAll(VDI.JRN_COALESCE) 

4420 if len(entries) == 0: 

4421 return False 

4422 elif len(entries) > 1: 

4423 raise util.SMException("More than one coalesce entry: " + str(entries)) 

4424 sr.scanLocked() 

4425 coalescedUuid = entries.popitem()[0] 

4426 garbage = sr.findGarbage() 

4427 for vdi in garbage: 

4428 if vdi.uuid == coalescedUuid: 

4429 return True 

4430 return False 

4431 

4432 

4433def get_coalesceable_leaves(session, srUuid, vdiUuids): 

4434 coalesceable = [] 

4435 sr = SR.getInstance(srUuid, session) 

4436 sr.scanLocked() 

4437 for uuid in vdiUuids: 

4438 vdi = sr.getVDI(uuid) 

4439 if not vdi: 

4440 raise util.SMException("VDI %s not found" % uuid) 

4441 if vdi.isLeafCoalesceable(): 

4442 coalesceable.append(uuid) 

4443 return coalesceable 

4444 

4445 

4446def cache_cleanup(session, srUuid, maxAge): 

4447 sr = SR.getInstance(srUuid, session) 

4448 return sr.cleanupCache(maxAge) 

4449 

4450 

4451def debug(sr_uuid, cmd, vdi_uuid): 

4452 Util.log("Debug command: %s" % cmd) 

4453 sr = SR.getInstance(sr_uuid, None) 

4454 if not isinstance(sr, LVMSR): 

4455 print("Error: not an LVHD SR") 

4456 return 

4457 sr.scanLocked() 

4458 vdi = sr.getVDI(vdi_uuid) 

4459 if not vdi: 

4460 print("Error: VDI %s not found") 

4461 return 

4462 print("Running %s on SR %s" % (cmd, sr)) 

4463 print("VDI before: %s" % vdi) 

4464 if cmd == "activate": 

4465 vdi._activate() 

4466 print("VDI file: %s" % vdi.path) 

4467 if cmd == "deactivate": 

4468 ns = NS_PREFIX_LVM + sr.uuid 

4469 sr.lvmCache.deactivate(ns, vdi.uuid, vdi.fileName, False) 

4470 if cmd == "inflate": 

4471 vdi.inflateFully() 

4472 sr.cleanup() 

4473 if cmd == "deflate": 

4474 vdi.deflate() 

4475 sr.cleanup() 

4476 sr.scanLocked() 

4477 print("VDI after: %s" % vdi) 

4478 

4479 

4480def abort_optional_reenable(uuid): 

4481 print("Disabling GC/coalesce for %s" % uuid) 

4482 ret = _abort(uuid) 

4483 input("Press enter to re-enable...") 

4484 print("GC/coalesce re-enabled") 

4485 lockGCRunning.release() 

4486 if ret: 

4487 lockGCActive.release() 

4488 

4489 

4490############################################################################## 

4491# 

4492# CLI 

4493# 

4494def main(): 

4495 action = "" 

4496 maxAge = 0 

4497 uuid = "" 

4498 background = False 

4499 force = False 

4500 dryRun = False 

4501 debug_cmd = "" 

4502 vdi_uuid = "" 

4503 shortArgs = "gGc:aqxu:bfdt:v:" 

4504 longArgs = ["gc", "gc_force", "clean_cache", "abort", "query", "disable", 

4505 "uuid=", "background", "force", "dry-run", "debug=", "vdi_uuid="] 

4506 

4507 try: 

4508 opts, args = getopt.getopt(sys.argv[1:], shortArgs, longArgs) 

4509 except getopt.GetoptError: 

4510 usage() 

4511 for o, a in opts: 

4512 if o in ("-g", "--gc"): 

4513 action = "gc" 

4514 if o in ("-G", "--gc_force"): 

4515 action = "gc_force" 

4516 if o in ("-c", "--clean_cache"): 

4517 action = "clean_cache" 

4518 maxAge = int(a) 

4519 if o in ("-a", "--abort"): 

4520 action = "abort" 

4521 if o in ("-q", "--query"): 

4522 action = "query" 

4523 if o in ("-x", "--disable"): 

4524 action = "disable" 

4525 if o in ("-u", "--uuid"): 

4526 uuid = a 

4527 if o in ("-b", "--background"): 

4528 background = True 

4529 if o in ("-f", "--force"): 

4530 force = True 

4531 if o in ("-d", "--dry-run"): 

4532 Util.log("Dry run mode") 

4533 dryRun = True 

4534 if o in ("-t", "--debug"): 

4535 action = "debug" 

4536 debug_cmd = a 

4537 if o in ("-v", "--vdi_uuid"): 

4538 vdi_uuid = a 

4539 

4540 if not action or not uuid: 

4541 usage() 

4542 if action == "debug" and not (debug_cmd and vdi_uuid) or \ 

4543 action != "debug" and (debug_cmd or vdi_uuid): 

4544 usage() 

4545 

4546 if action != "query" and action != "debug": 

4547 print("All output goes to log") 

4548 

4549 if action == "gc": 

4550 gc(None, uuid, background, dryRun) 

4551 elif action == "gc_force": 

4552 gc_force(None, uuid, force, dryRun, True) 

4553 elif action == "clean_cache": 

4554 cache_cleanup(None, uuid, maxAge) 

4555 elif action == "abort": 

4556 abort(uuid) 

4557 elif action == "query": 

4558 print("Currently running: %s" % get_state(uuid)) 

4559 elif action == "disable": 

4560 abort_optional_reenable(uuid) 

4561 elif action == "debug": 

4562 debug(uuid, debug_cmd, vdi_uuid) 

4563 

4564 

4565if __name__ == '__main__': 4565 ↛ 4566line 4565 didn't jump to line 4566, because the condition on line 4565 was never true

4566 main()