Coverage for drivers/qcow2util.py: 26%

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1#!/usr/bin/env python3

5# This program is free software: you can redistribute it and/or modify

6# it under the terms of the GNU General Public License as published by

7# the Free Software Foundation, either version 3 of the License, or

8# (at your option) any later version.

9# This program is distributed in the hope that it will be useful,

10# but WITHOUT ANY WARRANTY; without even the implied warranty of

11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

12# GNU General Public License for more details.

13#

14# You should have received a copy of the GNU General Public License

15# along with this program. If not, see <https://www.gnu.org/licenses/>.

17from sm_typing import Any, Callable, Dict, Final, List, Optional, Tuple, cast, override

18from typing import BinaryIO

20import errno

21import os

22import re

23import time

24import struct

25import zlib

26import json

27from pathlib import Path

29import util

30import xs_errors

31from blktap2 import TapCtl

32from cowutil import CowUtil, CowImageInfo

33from lvmcache import LVMCache

34from constants import NS_PREFIX_LVM, VG_PREFIX

36# ------------------------------------------------------------------------------

38MAX_QCOW_CHAIN_LENGTH: Final = 30

40QCOW2_DEFAULT_CLUSTER_SIZE: Final = 64 * 1024 # 64 KiB

42MIN_QCOW_SIZE: Final = QCOW2_DEFAULT_CLUSTER_SIZE

44MAX_QCOW_SIZE: Final = 17589500641280 # Max size so that the fully allocated size with metadata is under the max size of EXT4 (17592185061376 bytes fully allocated)

46QEMU_IMG: Final = "/usr/bin/qemu-img"

47QCOW2_HELPER = "/opt/xensource/libexec/qcow2_helper"

49QCOW2_TYPE: Final = "qcow2"

50RAW_TYPE: Final = "raw"

52# ------------------------------------------------------------------------------

54class QCowUtil(CowUtil):

56 # We followed specifications found here:

57 # https://github.com/qemu/qemu/blob/master/docs/interop/qcow2.txt

59 QCOW2_MAGIC = 0x514649FB # b"QFI\xfb": Magic number for QCOW2 files

60 QCOW2_HEADER_SIZE = 104 # In fact the last information we need is at offset 40-47

61 QCOW2_L2_SIZE = QCOW2_DEFAULT_CLUSTER_SIZE

62 QCOW2_BACKING_FILE_OFFSET = 8

64 ALLOCATED_ENTRY_BIT = (

65 0x8000_0000_0000_0000 # Bit 63 is the allocated bit for standard cluster

66 )

67 CLUSTER_TYPE_BIT = 0x4000_0000_0000_0000 # 0 for standard, 1 for compressed cluster

68 L2_OFFSET_MASK = 0x00FF_FFFF_FFFF_FF00 # Bits 9-55 are offset of L2 table.

69 CLUSTER_DESCRIPTION_MASK = 0x3FFF_FFFF_FFFF_FFFF # Bit 0-61 is cluster description

70 STANDARD_CLUSTER_OFFSET_MASK = (

71 0x00FF_FFFF_FFFF_FF00 # Bits 9-55 are offset of standard cluster

72 )

74 def __init__(self):

75 self.qcow_read = False

77 def _read_qcow2(self, path: str, read_clusters: bool = False):

78 phys_disk_size = self.getSizePhys(path)

79 with open(path, "rb") as qcow2_file:

80 self.filename = path # Keep the filename if clean is called

81 self.header = self._read_qcow2_header(qcow2_file)

82 if read_clusters:

83 self.l1 = self._get_l1_entries(qcow2_file)

84 # The l1_to_l2 allows to get L2 entries for a given L1. If L1 entry

85 # is not allocated we store an empty list.

86 self.l1_to_l2: Dict[int, List[int]] = {}

88 for l1_entry in self.l1:

89 l2_offset = l1_entry & self.L2_OFFSET_MASK

90 if l2_offset == 0:

91 self.l1_to_l2[l1_entry] = []

92 elif l2_offset > phys_disk_size: #TODO: This sometime happen for a correct VDI (while coalescing online?)

93 raise xs_errors.XenError("VDISize", "L2 Offset is bigger than physical disk {}".format(path))

94 else:

95 self.l1_to_l2[l1_entry] = self._get_l2_entries(

96 qcow2_file, l2_offset

97 )

98 self.qcow_read = True

100 def _get_l1_entries(self, file: BinaryIO) -> List[int]:

101 """Returns the list of all L1 entries.

102

103 Args:

104 file: The qcow2 file object.

105

106 Returns:

107 list: List of all L1 entries

108 """

109 l1_table_offset = self.header["l1_table_offset"]

110 file.seek(l1_table_offset)

111

112 l1_table_size = self.header["l1_size"] * 8 # Each L1 entry is 8 bytes

113 l1_table = file.read(l1_table_size)

114

115 return [

116 struct.unpack(">Q", l1_table[i : i + 8])[0]

117 for i in range(0, len(l1_table), 8)

118 ]

119

120 @staticmethod

121 def _get_l2_entries(file: BinaryIO, l2_offset: int) -> List[int]:

122 """Returns the list of all L2 entries at a given L2 offset.

123

124 Args:

125 file: The qcow2 file.

126 l2_offset: the L2 offset where to look for entries

127

128 Returns:

129 list: List of all L2 entries

130 """

131 # The size of L2 is 65536 bytes and each entry is 8 bytes.

132 file.seek(l2_offset)

133 l2_table = file.read(QCowUtil.QCOW2_L2_SIZE)

134

135 return [

136 struct.unpack(">Q", l2_table[i : i + 8])[0]

137 for i in range(0, len(l2_table), 8)

138 ]

139

140 @staticmethod

141 def _read_qcow2_backingfile(file: BinaryIO, backing_file_offset: int , backing_file_size: int) -> str:

142 if backing_file_offset == 0:

143 return ""

144

145 file.seek(backing_file_offset)

146 parent_name = file.read(backing_file_size)

147 return parent_name.decode("UTF-8")

148

149 @staticmethod

150 def _read_qcow2_header(file: BinaryIO) -> Dict[str, Any]:

151 """Returns a dict containing some information from QCow2 header.

152

153 Args:

154 file: The qcow2 file object.

155

156 Returns:

157 dict: magic, version, cluster_bits, l1_size and l1_table_offset.

158

159 Raises:

160 ValueError: if qcow2 magic is not recognized or cluster size not supported.

161 """

162 # The header is as follow:

163 #

164 # magic: u32, // Magic string "QFI\xfb"

165 # version: u32, // Version (2 or 3)

166 # backing_file_offset: u64, // Offset to the backing file name

167 # backing_file_size: u32, // Size of the backing file name

168 # cluster_bits: u32, // Bits used for addressing within a cluster

169 # size: u64, // Virtual disk size

170 # crypt_method: u32, // 0 = no encryption, 1 = AES encryption

171 # l1_size: u32, // Number of entries in the L1 table

172 # l1_table_offset: u64, // Offset to the active L1 table

173 # refcount_table_offset: u64, // Offset to the refcount table

174 # refcount_table_clusters: u32, // Number of clusters for the refcount table

175 # nb_snapshots: u32, // Number of snapshots in the image

176 # snapshots_offset: u64, // Offset to the snapshot table

177

178 file.seek(0)

179 header = file.read(QCowUtil.QCOW2_HEADER_SIZE)

180 (

181 magic,

182 version,

183 backing_file_offset,

184 backing_file_size,

185 cluster_bits,

186 size,

187 _,

188 l1_size,

189 l1_table_offset,

190 refcount_table_offset,

191 _,

192 _,

193 snapshots_offset,

194 ) = struct.unpack(">IIQIIQIIQQIIQ", header[:72])

195

196 if magic != QCowUtil.QCOW2_MAGIC:

197 raise ValueError("Not a valid QCOW2 file")

198

199 parent_name = QCowUtil._read_qcow2_backingfile(file, backing_file_offset, backing_file_size)

200

201 return {

202 "version": version,

203 "backing_file_offset": backing_file_offset,

204 "backing_file_size": backing_file_size,

205 "virtual_disk_size": size,

206 "cluster_bits": cluster_bits,

207 "l1_size": l1_size,

208 "l1_table_offset": l1_table_offset,

209 "refcount_table_offset": refcount_table_offset,

210 "snapshots_offset": snapshots_offset,

211 "parent": parent_name,

212 }

213

214 @staticmethod

215 def _is_l1_allocated(entry: int) -> bool:

216 """Checks if the given L1 entry is allocated.

217

218 If the offset is 0 then the L2 table and all clusters described

219 by this L2 table are unallocated.

220

221 Args:

222 entry: L1 entry

223

224 Returns:

225 bool: True if the L1 entry is allocated (ie has a valid offset).

226 False otherwise.

227 """

228 return (entry & QCowUtil.L2_OFFSET_MASK) != 0

229

230 @staticmethod

231 def _is_l2_allocated(entry: int) -> bool:

232 """Checks if a given entry is allocated.

233

234 Currently we only support standard clusters. And for standard clusters

235 the bit 63 is set to 1 for allocated ones or offset is not 0.

236

237 Args:

238 entry: L2 entry

239

240 Returns:

241 bool: Returns True if the L2 entry is allocated, False otherwise

242

243 Raises:

244 raise an exception if the cluster is not a standard one.

245 """

246 assert entry & QCowUtil.CLUSTER_TYPE_BIT == 0

247 return (entry & QCowUtil.ALLOCATED_ENTRY_BIT != 0) or (

248 entry & QCowUtil.STANDARD_CLUSTER_OFFSET_MASK != 0

249 )

250

251 @staticmethod

252 def _get_allocated_clusters(l2_entries: List[int]) -> List[int]:

253 """Get all allocated clusters in a given list of L2 entries.

254

255 Args:

256 l2_entries: A list of L2 entries.

257

258 Returns:

259 A list of all allocated entries

260 """

261 return [entry for entry in l2_entries if QCowUtil._is_l2_allocated(entry)]

262

263 @staticmethod

264 def _get_cluster_to_byte(clusters: int, cluster_bits: int) -> int:

265 # (1 << cluster_bits) give cluster size in byte

266 return clusters * (1 << cluster_bits)

267

268 def _get_number_of_allocated_clusters(self) -> int:

269 """Get the number of allocated clusters.

270

271 Args:

272 self: A QcowInfo object.

273

274 Returns:

275 An integer that is the list of allocated clusters.

276 """

277 assert(self.qcow_read)

278

279 allocated_clusters = 0

280

281 for l2_entries in self.l1_to_l2.values():

282 allocated_clusters += len(self._get_allocated_clusters(l2_entries))

283

284 return allocated_clusters

285

286 @staticmethod

287 def _move_backing_file(

288 f: BinaryIO, old_offset: int, new_offset: int, data_size: int

289 ) -> None:

290 """Move a number of bytes from old_offset to new_offset and replaces the old

291 value by 0s. It is up to the caller to save the current position in the file

292 if needed.

293

294 Args:

295 f: the file the will be modified

296 old_offset: the current offset

297 new_offset: the new offset where we want to move data

298 data_size: Size in bytes of data that we want to move

299

300 Returns:

301 Nothing but the file f is modified and the position in the file also.

302 """

303 # Read the string at backing_file_offset

304 f.seek(old_offset)

305 data = f.read(data_size)

306

307 # Write zeros at the original location

308 f.seek(old_offset)

309 f.write(b"\x00" * data_size)

310

311 # Write the string to the new location

312 f.seek(new_offset)

313 f.write(data)

314

315 def _add_or_find_custom_header(self) -> int:

316 """Add custom header at the end of header extensions

317

318 It finds the end of the header extensions and add the custom header.

319 If the header already exists nothing is done.

320

321 Args:

322

323 Returns:

324 It returns the data offset where custom header is found or created.

325 If data offset is 0 something weird happens.

326 The qcow2 file in self.filename can be modified.

327 """

328 assert self.qcow_read

329

330 header_length = 72 # This is the default value for version 2 images

331

332 custom_header_type = 0x76617465 # vate: it is easy to recognize with hexdump -C

333 custom_header_length = 8

334 custom_header_data = 0

335 # We don't need padding because we are already aligned

336 custom_header = struct.pack(

337 ">IIQ", custom_header_type, custom_header_length, custom_header_data

338 )

339

340 with open(self.filename, "rb+") as qcow2_file:

341 if self.header["version"] == 3:

342 qcow2_file.seek(100) # 100 is the offset of header_length

343 header_length = int.from_bytes(qcow2_file.read(4), "big")

344

345 # After the image header we found Header extension. So we need to find the end of

346 # the header extension area and add our custom header.

347 qcow2_file.seek(header_length)

348

349 custom_data_offset = 0

350

351 while True:

352 ext_type = int.from_bytes(qcow2_file.read(4), "big")

353 ext_len = int.from_bytes(qcow2_file.read(4), "big")

354

355 if ext_type == custom_header_type:

356 # A custom header is already there

357 custom_data_offset = qcow2_file.tell()

358 break

359

360 if ext_type == 0x00000000:

361 # End mark found. If we found the end mark it means that we didn't find

362 # the custom header. So we need to add it.

363 custom_data_offset = qcow2_file.tell()

364

365 # We will overwrite the end marker so rewind a little bit to

366 # write the new type extension and the new length. But if there is

367 # a backing file we need to move it to make some space.

368 if self.header["backing_file_offset"]:

369 # Keep current position

370 saved_pos = qcow2_file.tell()

371

372 bf_offset = self.header["backing_file_offset"]

373 bf_size = self.header["backing_file_size"]

374 bf_new_offset = bf_offset + len(custom_header)

375 self._move_backing_file(

376 qcow2_file, bf_offset, bf_new_offset, bf_size

377 )

378

379 # Update the header to match the new backing file offset

380 self.header["backing_file_offset"] = bf_new_offset

381 qcow2_file.seek(self.QCOW2_BACKING_FILE_OFFSET)

382 qcow2_file.write(struct.pack(">Q", bf_new_offset))

383

384 # Restore saved position

385 qcow2_file.seek(saved_pos)

386

387 qcow2_file.seek(-8, 1)

388 qcow2_file.write(custom_header)

389 break

390

391 # Round up the header extension size to the next multiple of 8

392 ext_len = (ext_len + 7) & 0xFFFFFFF8

393 qcow2_file.seek(ext_len, 1)

394

395 return custom_data_offset

396

397 def _set_l1_zero(self):

398 zero = int(0).to_bytes(1, "little")

399 nb_of_entries_per_cluster = QCOW2_DEFAULT_CLUSTER_SIZE/8

400 return list(zero * int(nb_of_entries_per_cluster/8))

401

402 def _set_l2_zero(self, b, i):

403 return b & ~(1 << i)

404

405 def _set_l2_one(self, b, i):

406 return b | (1 << i)

407

408 def _create_bitmap(self) -> bytes:

409 idx: int = 0

410 bitmap = list()

411 b = 0

412 for l1_entry in self.l1:

413 if not self._is_l1_allocated(l1_entry):

414 bitmap.extend(self._set_l1_zero())

415 continue

416

417 l2_table = self.l1_to_l2[l1_entry] #L2 is cluster_size/8 entries of cluster_size page

418 for l2_entry in l2_table:

419 if self._is_l2_allocated(l2_entry):

420 b = self._set_l2_one(b, idx)

421 else:

422 b = self._set_l2_zero(b, idx)

423 idx += 1

424 if idx == 8:

425 bitmap.append(b)

426 b = 0

427 idx = 0

428 return struct.pack("B"*len(bitmap), *bitmap)

429

430 # ----

431 # Implementation of CowUtil

432 # ----

433

434 @override

435 def getMinImageSize(self) -> int:

436 return MIN_QCOW_SIZE

437

438 @override

439 def getMaxImageSize(self) -> int:

440 return MAX_QCOW_SIZE

441

442 @override

443 def getBlockSize(self, path: str) -> int:

444 self._read_qcow2(path)

445 return 1 << self.header["cluster_bits"]

446

447 @override

448 def getFooterSize(self) -> int:

449 return 0

450

451 @override

452 def getDefaultPreallocationSizeVirt(self) -> int:

453 """vhdutil answer max size (2TiB) here but we don't want to allocate for max size in QCOW2, it would make small LV a lot bigger."""

454 return MIN_QCOW_SIZE

455

456 @override

457 def getMaxChainLength(self) -> int:

458 return MAX_QCOW_CHAIN_LENGTH

459

460 @override

461 def calcOverheadEmpty(self, virtual_size: int, block_size: Optional[int] = None) -> int:

462 if block_size:

463 cluster_size = block_size

464 else:

465 cluster_size = QCOW2_DEFAULT_CLUSTER_SIZE

466 cmd = [QEMU_IMG, "measure", "-O", "qcow2", "--output", "json", "-o", f"cluster_size={cluster_size}", "--size", f"{virtual_size}"]

467 output = json.loads(self._ioretry(cmd))

468 return int(output["required"])

469

470 @override

471 def calcOverheadBitmap(self, virtual_size: int) -> int:

472 return 0 #TODO: What do we send back?

473

474 @override

475 def getInfo(

476 self,

477 path: str,

478 extractUuidFunction: Callable[[str], str],

479 includeParent: bool = True,

480 resolveParent: bool = True,

481 useBackupFooter: bool = False

482 ) -> CowImageInfo:

483 #TODO: handle resolveParent

484 self._read_qcow2(path)

485 uuid = extractUuidFunction(path)

486 cowinfo = CowImageInfo(uuid)

487 cowinfo.path = path

488 cowinfo.sizeVirt = self.header["virtual_disk_size"]

489 cowinfo.sizePhys = self.getSizePhys(path)

490 cowinfo.hidden = self.getHidden(path)

491 if includeParent:

492 parent_path = self.header["parent"]

493 if parent_path != "":

494 cowinfo.parentPath = parent_path

495 cowinfo.parentUuid = extractUuidFunction(parent_path)

496

497 return cowinfo

498

499 @override

500 def getInfoFromLVM(

501 self, lvName: str, extractUuidFunction: Callable[[str], str], vgName: str

502 ) -> Optional[CowImageInfo]:

503 lvcache = LVMCache(vgName)

504 return self._getInfoLV(lvcache, extractUuidFunction, vgName, lvName)

505

506 def _getInfoLV(

507 self, lvcache: LVMCache, extractUuidFunction: Callable[[str], str], vgName: str, lvName: str

508 ) -> Optional[CowImageInfo]:

509 lvPath = "/dev/{}/{}".format(vgName, lvName)

510 lvcache.refresh()

511 if lvName not in lvcache.lvs:

512 util.SMlog("{} does not exist anymore".format(lvName))

513 return None

514

515 vdiUuid = extractUuidFunction(lvPath)

516 srUuid = vgName.replace(VG_PREFIX, "")

517

518 ns = NS_PREFIX_LVM + srUuid

519 lvcache.activate(ns, vdiUuid, lvName, False)

520 try:

521 cowinfo = self.getInfo(lvPath, extractUuidFunction)

522 finally:

523 lvcache.deactivate(ns, vdiUuid, lvName, False)

524 return cowinfo

525

526 @override

527 def getAllInfoFromVG(

528 self,

529 pattern: str,

530 extractUuidFunction: Callable[[str], str],

531 vgName: Optional[str] = None,

532 parents: bool = False,

533 exitOnError: bool = False

534 ) -> Dict[str, CowImageInfo]:

535 result: Dict[str, CowImageInfo] = dict()

536 #TODO: handle exitOnError

537 if vgName: 537 ↛ 538line 537 didn't jump to line 538, because the condition on line 537 was never true

538 reg = re.compile(pattern)

539 lvcache = LVMCache(vgName)

540 lvcache.refresh()

541 # We get size in lvcache.lvs[lvName].size (in bytes)

542 # We could read the header from the PV directly

543 lvList = list(lvcache.lvs.keys())

544 for lvName in lvList:

545 # lvinfo = lvcache.lvs[lvName]

546 if reg.match(lvName):

547 cowinfo = self._getInfoLV(lvcache, extractUuidFunction, vgName, lvName)

548 if cowinfo is None: #We get None if the LV stopped existing in the meanwhile

549 continue

550 cowinfo.path = lvName # Function CowUtil.getParentChain expect lvName here, otherwise blktap.{_activate,_deactivate} crashes

551 result[cowinfo.uuid] = cowinfo

552 if parents:

553 parentUuid = cowinfo.parentUuid

554 parentPath = cowinfo.parentPath

555 while parentUuid != "":

556 parentLvName = parentPath.split("/")[-1]

557 parent_cowinfo = self._getInfoLV(lvcache, extractUuidFunction, vgName, parentLvName)

558 if parent_cowinfo is None: #Parent disappeared while scanning

559 raise util.SMException("Parent of {} wasn't found during scan".format(lvName))

560 parentUuid = parent_cowinfo.parentUuid

561 parentPath = parent_cowinfo.parentPath

562 parent_cowinfo.path = parentLvName #Same reason as above, some users expect LvName here instead of path

563 result[parent_cowinfo.uuid] = parent_cowinfo

564

565 return result

566 else:

567 pattern_p: Path = Path(pattern)

568 list_qcow = list(pattern_p.parent.glob(pattern_p.name))

569 for qcow in list_qcow: 569 ↛ 570line 569 didn't jump to line 570, because the loop on line 569 never started

570 qcow_str = str(qcow)

571 info = self.getInfo(qcow_str, extractUuidFunction)

572 result[info.uuid] = info

573 return result

574

575 @override

576 def getParent(self, path: str, extractUuidFunction: Callable[[str], str]) -> Optional[str]:

577 parent = self.getParentNoCheck(path)

578 if parent:

579 return extractUuidFunction(parent)

580 return None

581

582 @override

583 def getParentNoCheck(self, path: str) -> Optional[str]:

584 self._read_qcow2(path)

585 parent_path = self.header["parent"]

586 if parent_path == "":

587 return None

588 return parent_path

589

590 @override

591 def hasParent(self, path: str) -> bool:

592 if self.getParentNoCheck(path):

593 return True

594 return False

595

596 @override

597 def setParent(self, path: str, parentPath: str, parentRaw: bool) -> None:

598 pid_openers = util.get_openers_pid(path)

599 if pid_openers:

600 util.SMlog("Rebasing while process {} has the VDI opened".format(pid_openers))

601

602 parentType = QCOW2_TYPE

603 if parentRaw:

604 parentType = RAW_TYPE

605 cmd = [QEMU_IMG, "rebase", "-u", "-f", QCOW2_TYPE, "-F", parentType, "-b", parentPath, path]

606 self._ioretry(cmd)

607

608 @override

609 def getHidden(self, path: str) -> bool:

610 """Get hidden property according to the value b

611

612 Args:

613

614 Returns:

615 True if hidden is set, False otherwise

616 """

617 self._read_qcow2(path)

618 custom_data_offset = self._add_or_find_custom_header()

619 if custom_data_offset == 0:

620 raise util.SMException("Custom data offset not found... should not reach this")

621

622 with open(path, "rb") as qcow2_file:

623 qcow2_file.seek(custom_data_offset)

624 hidden = qcow2_file.read(1)

625 if hidden == b"\x00":

626 return False

627 return True

628

629 @override

630 def setHidden(self, path: str, hidden: bool = True) -> None:

631 """Set hidden property according to the value b

632

633 Args:

634 bool: True if you want to set the property. False otherwise

635

636 Returns:

637 nothing. If the custom headers is not found it is created so the

638 qcow file can be modified.

639 """

640 self._read_qcow2(path)

641 custom_data_offset = self._add_or_find_custom_header()

642 if custom_data_offset == 0:

643 raise util.SMException("Custom data offset not found... should not reach this")

644

645 with open(self.filename, "rb+") as qcow2_file:

646 qcow2_file.seek(custom_data_offset)

647 if hidden:

648 qcow2_file.write(b"\x01")

649 else:

650 qcow2_file.write(b"\x00")

651

652 @override

653 def getSizeVirt(self, path: str) -> int:

654 self._read_qcow2(path)

655 return self.header['virtual_disk_size']

656

657 @override

658 def setSizeVirt(self, path: str, size: int, jFile: str) -> None:

659 """

660 size: byte

661 jFile: a journal file used for resizing with VHD, not useful for QCOW2

662 """

663 cmd = [QEMU_IMG, "resize", path, str(size)]

664 self._ioretry(cmd)

665

666 @override

667 def setSizeVirtFast(self, path: str, size: int) -> None:

668 self.setSizeVirt(path, size, "")

669

670 @override

671 def getMaxResizeSize(self, path: str) -> int:

672 return 0

673

674 @override

675 def getSizePhys(self, path: str) -> int:

676 size = os.stat(path).st_size

677 if size == 0:

678 size = int(self._ioretry(["blockdev", "--getsize64", path]))

679 return size

680

681 @override

682 def setSizePhys(self, path: str, size: int, debug: bool = True) -> None:

683 pass #TODO: Doesn't exist for QCow2, do we need to use it?

684

685 @override

686 def getAllocatedSize(self, path: str) -> int:

687 cmd = [QCOW2_HELPER, "allocated", path]

688 return int(self._ioretry(cmd))

689

690 @override

691 def getResizeJournalSize(self) -> int:

692 return 0

693

694 @override

695 def killData(self, path: str) -> None:

696 """Remove all data and reset L1/L2 table.

697

698 Args:

699 self: The QcowInfo object.

700

701 Returns:

702 nothing.

703 """

704 self._read_qcow2(path, read_clusters=True)

705 # We need to reset L1 entries and then just truncate the file right

706 # after L1 entries

707 with open(self.filename, "r+b") as file:

708 l1_table_offset = self.header["l1_table_offset"]

709 file.seek(l1_table_offset)

710

711 l1_table_size = (

712 self.header["l1_size"] * 8

713 ) # size in bytes, each entry is 8 bytes

714 file.write(b"\x00" * l1_table_size)

715 file.truncate(l1_table_offset + l1_table_size)

716

717 @override

718 def getDepth(self, path: str) -> int:

719 cmd = [QEMU_IMG, "info", "--backing-chain", "--output=json", path]

720 ret = str(self._ioretry(cmd))

721 depth = len(re.findall("\"backing-filename\"", ret))+1

722 #chain depth is beginning at one for VHD, meaning a VHD without parent has depth = 1

723 return depth

724

725 @override

726 def getBlockBitmap(self, path: str) -> bytes:

727 cmd = [QCOW2_HELPER, "bitmap", path]

728 text = cast(bytes, self._ioretry(cmd, text=False))

729 return zlib.compress(text)

730

731 def _getTapdisk(self, path: str) -> Tuple[int, int]:

732 """

733 Return a tuple of (PID, Minor) for the given path

734 """

735 pid_openers = util.get_openers_pid(path)

736 if pid_openers:

737 if len(pid_openers) > 1:

738 raise xs_errors.XenError("Multiple openers for {}".format(path)) # TODO: There might be multiple PID? Yes, we can have the chain enabled for multiple leaf (i.e. after a clone), taken into account in cleanup.py

739 pid = pid_openers[0]

740 tapdiskList = TapCtl.list(pid=pid)

741 if len(tapdiskList) > 1: #TODO: There might more than one minor for this blktap?

742 raise xs_errors.XenError("TapdiskAlreadyRunning", "There is multiple minor for this tapdisk process")

743 minor = tapdiskList[0]["minor"]

744 return (pid, minor)

745 raise xs_errors.XenError("TapdiskFailed", "No tapdisk process found for {}".format(path))

746

747 @override

748 def coalesceOnline(self, path: str) -> int:

749 pid, minor = self._getTapdisk(path)

750 logger = util.LoggerCounter(10)

751

752 try:

753 TapCtl.commit(pid, minor, QCOW2_TYPE, path)

754 # We need to wait for query to return concluded

755 # We are technically ininterruptible since being interrupted will only stop checking if the job is done.

756 # We need to call `tap-ctl cancel` if we are interrupted, it is done in cleanup.py code.

757 except TapCtl.CommandFailure:

758 util.SMlog(f"Commit command failed on tapdisk instance {pid} (m: {minor}). Raising...")

759 raise

760

761 try:

762 status, nb, _ = TapCtl.query(pid, minor)

763 if status == "undefined":

764 util.SMlog("Tapdisk {} (m: {}) coalesce status undefined for {}".format(pid, minor, path))

765 return 0

766

767 while status != "concluded":

768 time.sleep(1)

769 status, nb, _ = TapCtl.query(pid, minor, quiet=True)

770 logger.log("Got status {} for tapdisk {} (m: {})".format(status, pid, minor))

771 return nb

772 except TapCtl.CommandFailure:

773 util.SMlog(f"Query command failed on tapdisk instance {pid} (m: {minor}). Raising...")

774 raise

775

776 @override

777 def cancelCoalesceOnline(self, path: str) -> None:

778 pid, minor = self._getTapdisk(path)

779

780 try:

781 TapCtl.cancel_commit(pid, minor)

782 except TapCtl.CommandFailure:

783 util.SMlog("Cancel command failed on tapdisk instance {}. Raising...".format(pid))

784 raise

785

786 @override

787 def coalesce(self, path: str) -> int:

788 # -d on commit make it not empty the original image since we don't intend to keep it

789 cmd = [QEMU_IMG, "commit", "-f", QCOW2_TYPE, path, "-d"]

790 ret = cast(str, self._ioretry(cmd)) # Allows to parse for byte coalesced, our qemu-img is supposed to be patched to output it.

791 lines = ret.splitlines()

792 if re.match("Image committed.", lines[-1]):

793 res_line = lines[-2]

794 else:

795 res_line = lines[-1]

796

797 results = re.match(r"\((\d+)/(\d+)\)", res_line)

798 if results:

799 committed_bytes = int(results.group(1))

800 return committed_bytes

801 raise xs_errors.XenError("TapdiskFailed", "Couldn't get commited size from qemu-img commit call") # TODO: We might not want to raise in this case, it would break if the qemu-img called isn't modified to print the coalesce result even if it succeeded in coalesceing

802

803 @override

804 def create(self, path: str, size: int, static: bool, msize: int = 0, block_size: Optional[int] = None) -> None:

805 cmd = [QEMU_IMG, "create", "-f", QCOW2_TYPE, path, str(size)]

806 if static:

807 cmd.extend(["-o", "preallocation=full"])

808 if block_size:

809 cmd.extend(["-o", f"cluster_size={str(block_size)}"])

810 self._ioretry(cmd)

811 self.setHidden(path, False) #We add hidden header at creation

812

813 @override

814 def snapshot(

815 self,

816 path: str,

817 parent: str,

818 parentRaw: bool,

819 msize: int = 0,

820 checkEmpty: bool = True,

821 is_mirror_image: bool = False

822 ) -> None:

823 # TODO: msize, it's use to preallocate metadata, could we honor this too?

824 # TODO: checkEmpty? If it is False, then the parent could be empty and should still be used for snapshot

825 # But if True, if the parent is empty, we do what? vhd would just use the parent of parent as base, should we emulate this behavior?

826

827 cmd = [QEMU_IMG, "create"]

828

829 if parentRaw:

830 parent_type = RAW_TYPE

831 cluster_size = QCOW2_DEFAULT_CLUSTER_SIZE

832 else:

833 parent_type = QCOW2_TYPE

834 cluster_size = self.getBlockSize(parent)

835 args = ["-f", QCOW2_TYPE, "-F", parent_type, "-b", parent]

836

837 if is_mirror_image:

838 # is_mirror_image override the cluster size to ensure that we have a write of 512b to avoid having to read the parent during a migration.

839 # This is needed because the blkif blocksize is only 512b, as such it will try to only write blocks smaller than the cluster size.

840 # To write a smaller block, we would need to read the parent image cluster then change the 512b block.

841 # The parent being empty during the mirroring phase, reading from it would read zeros and corrupt the cluster.

842 # It also enable extended_l2 for this purpose, this is only done in the snapshot used for the mirror, this configuration will be lost when coalesced in its parent.

843 # Ensuring we go back to a better cluster_size for performance reasons.

844 # This limit our images max size to 64TiB.

845 cluster_size = 16 * 1024 # 16KiB

846 args.extend(["-o", "extended_l2=on"])

847

848 args.extend(["-o", f"cluster_size={cluster_size}"])

849 cmd.extend(args)

850 cmd.append(path)

851

852 self._ioretry(cmd)

853 self.setHidden(path, False) #We add hidden header at creation

854