1#!/usr/bin/env python3 

2# 

3# Copyright (C) 2020 Vates SAS - ronan.abhamon@vates.fr 

4# 

5# This program is free software: you can redistribute it and/or modify 

6# it under the terms of the GNU General Public License as published by 

7# the Free Software Foundation, either version 3 of the License, or 

8# (at your option) any later version. 

9# This program is distributed in the hope that it will be useful, 

10# but WITHOUT ANY WARRANTY; without even the implied warranty of 

11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

12# GNU General Public License for more details. 

13# 

14# You should have received a copy of the GNU General Public License 

15# along with this program. If not, see <https://www.gnu.org/licenses/>. 

16# 

17 

18from sm_typing import ( 

19 Any, 

20 Dict, 

21 List, 

22 cast, 

23 override, 

24) 

25 

26import json 

27import linstor 

28import os.path 

29import re 

30import shutil 

31import socket 

32import stat 

33import time 

34import util 

35import uuid 

36 

37# Persistent prefix to add to RAW persistent volumes. 

38PERSISTENT_PREFIX = 'xcp-persistent-' 

39 

40# Contains the data of the "/var/lib/linstor" directory. 

41DATABASE_VOLUME_NAME = PERSISTENT_PREFIX + 'database' 

42DATABASE_SIZE = 1 << 30 # 1GB. 

43DATABASE_PATH = '/var/lib/linstor' 

44DATABASE_MKFS = 'mkfs.ext4' 

45 

46LINSTOR_SATELLITE_PORT = 3366 

47 

48REG_DRBDADM_PRIMARY = re.compile("([^\\s]+)\\s+role:Primary") 

49REG_DRBDSETUP_IP = re.compile('[^\\s]+\\s+(.*):.*$') 

50 

51DRBD_BY_RES_PATH = '/dev/drbd/by-res/' 

52 

53PLUGIN = 'linstor-manager' 

54 

55LinstorLocalVolumeOpeners = Dict[str, Dict[str, Any]] 

56LinstorVolumeOpeners = Dict[str, LinstorLocalVolumeOpeners] 

57 

58# ============================================================================== 

59 

60def get_local_volume_openers(resource_name, volume) -> LinstorLocalVolumeOpeners: 

61 if not resource_name or volume is None: 

62 raise Exception('Cannot get DRBD openers without resource name and/or volume.') 

63 

64 path = '/sys/kernel/debug/drbd/resources/{}/volumes/{}/openers'.format( 

65 resource_name, volume 

66 ) 

67 

68 with open(path, 'r') as openers: 

69 # Not a big cost, so read all lines directly. 

70 lines = openers.readlines() 

71 

72 result = {} 

73 

74 opener_re = re.compile('(.*)\\s+([0-9]+)\\s+([0-9]+)') 

75 for line in lines: 

76 match = opener_re.match(line) 

77 assert match 

78 

79 groups = match.groups() 

80 process_name = groups[0] 

81 pid = groups[1] 

82 open_duration_ms = groups[2] 

83 

84 try: 

85 cmdline = util.get_process_cmdline(int(pid)) 

86 except Exception as e: 

87 util.SMlog(f"Failed to get command line of `{pid}`: {e}") 

88 cmdline = [] 

89 

90 result[pid] = { 

91 'process-name': process_name, 

92 'open-duration': open_duration_ms, 

93 'cmdline': cmdline 

94 } 

95 

96 return cast(LinstorLocalVolumeOpeners, json.dumps(result)) 

97 

98def get_all_volume_openers(resource_name, volume) -> LinstorVolumeOpeners: 

99 PLUGIN_CMD = 'getDrbdOpeners' 

100 

101 volume = str(volume) 

102 openers = {} 

103 

104 session = util.get_localAPI_session() 

105 

106 hosts = session.xenapi.host.get_all_records() 

107 for host_ref, host_record in hosts.items(): 

108 node_name = host_record['hostname'] 

109 try: 

110 if not session.xenapi.host_metrics.get_record( 

111 host_record['metrics'] 

112 )['live']: 

113 # Ensure we call plugin on online hosts only. 

114 continue 

115 

116 openers[node_name] = json.loads( 

117 session.xenapi.host.call_plugin(host_ref, PLUGIN, PLUGIN_CMD, { 

118 'resourceName': resource_name, 

119 'volume': volume 

120 }) 

121 ) 

122 except Exception as e: 

123 util.SMlog('Failed to get openers of `{}` on `{}`: {}'.format( 

124 resource_name, node_name, e 

125 )) 

126 

127 return openers 

128 

129 

130# ============================================================================== 

131 

132def round_up(value, divisor): 

133 assert divisor 

134 divisor = int(divisor) 

135 return ((int(value) + divisor - 1) // divisor) * divisor 

136 

137 

138def round_down(value, divisor): 

139 assert divisor 

140 value = int(value) 

141 return value - (value % int(divisor)) 

142 

143 

144# ============================================================================== 

145 

146def _get_controller_addresses() -> List[str]: 

147 try: 

148 (ret, stdout, stderr) = util.doexec([ 

149 "/usr/sbin/ss", "-tnpH", "state", "established", f"( sport = :{LINSTOR_SATELLITE_PORT} )" 

150 ]) 

151 if ret == 0: 

152 return [ 

153 line.split()[3].rsplit(":", 1)[0] 

154 for line in stdout.splitlines() 

155 ] 

156 util.SMlog(f"Unexpected code {ret}: {stderr}") 

157 except Exception as e: 

158 util.SMlog(f"Unable to get controller addresses: {e}") 

159 return [] 

160 

161def _get_controller_uri() -> str: 

162 # TODO: Check that an IP address from the current pool is returned. 

163 addresses = _get_controller_addresses() 

164 return "linstor://" + addresses[0] if addresses else "" 

165 

166def get_controller_uri(): 

167 retries = 0 

168 while True: 

169 uri = _get_controller_uri() 

170 if uri: 

171 return uri 

172 

173 retries += 1 

174 if retries >= 30: 

175 break 

176 time.sleep(1) 

177 

178 

179def get_controller_node_name(): 

180 PLUGIN_CMD = 'hasControllerRunning' 

181 

182 (ret, stdout, stderr) = util.doexec([ 

183 'drbdadm', 'status', DATABASE_VOLUME_NAME 

184 ]) 

185 

186 if ret == 0: 

187 if stdout.startswith('{} role:Primary'.format(DATABASE_VOLUME_NAME)): 

188 return 'localhost' 

189 

190 res = REG_DRBDADM_PRIMARY.search(stdout) 

191 if res: 

192 return res.groups()[0] 

193 

194 session = util.timeout(5, util.get_localAPI_session) 

195 

196 for host_ref, host_record in session.xenapi.host.get_all_records().items(): 

197 node_name = host_record['hostname'] 

198 try: 

199 if not session.xenapi.host_metrics.get_record( 

200 host_record['metrics'] 

201 )['live']: 

202 continue 

203 

204 if util.strtobool(session.xenapi.host.call_plugin( 

205 host_ref, PLUGIN, PLUGIN_CMD, {} 

206 )): 

207 return node_name 

208 except Exception as e: 

209 util.SMlog('Failed to call plugin to get controller on `{}`: {}'.format( 

210 node_name, e 

211 )) 

212 

213 

214def demote_drbd_resource(node_name, resource_name): 

215 PLUGIN_CMD = 'demoteDrbdResource' 

216 

217 session = util.timeout(5, util.get_localAPI_session) 

218 

219 for host_ref, host_record in session.xenapi.host.get_all_records().items(): 

220 if host_record['hostname'] != node_name: 

221 continue 

222 

223 try: 

224 session.xenapi.host.call_plugin( 

225 host_ref, PLUGIN, PLUGIN_CMD, {'resource_name': resource_name} 

226 ) 

227 except Exception as e: 

228 util.SMlog('Failed to demote resource `{}` on `{}`: {}'.format( 

229 resource_name, node_name, e 

230 )) 

231 raise Exception( 

232 'Can\'t demote resource `{}`, unable to find node `{}`' 

233 .format(resource_name, node_name) 

234 ) 

235 

236# ============================================================================== 

237 

238class LinstorVolumeManagerError(Exception): 

239 ERR_GENERIC = 0, 

240 ERR_VOLUME_EXISTS = 1, 

241 ERR_VOLUME_NOT_EXISTS = 2, 

242 ERR_VOLUME_DESTROY = 3, 

243 ERR_GROUP_NOT_EXISTS = 4, 

244 ERR_VOLUME_IN_USE = 5 

245 

246 def __init__(self, message, code=ERR_GENERIC): 

247 super(LinstorVolumeManagerError, self).__init__(message) 

248 self._code = code 

249 

250 @property 

251 def code(self): 

252 return self._code 

253 

254 

255# ============================================================================== 

256 

257# Note: 

258# If a storage pool is not accessible after a network change: 

259# linstor node interface modify <NODE> default --ip <IP> 

260 

261 

262class LinstorVolumeManager(object): 

263 """ 

264 API to manager LINSTOR volumes in XCP-ng. 

265 A volume in this context is a physical part of the storage layer. 

266 """ 

267 

268 __slots__ = ( 

269 '_linstor', '_uri', '_logger', '_redundancy', 

270 '_base_group_name', '_group_name', '_ha_group_name', 

271 '_volumes', '_storage_pools', '_storage_pools_time', 

272 '_kv_cache', '_resource_cache', '_volume_info_cache', 

273 '_kv_cache_dirty', '_resource_cache_dirty', '_volume_info_cache_dirty', 

274 '_resources_info_cache', 

275 ) 

276 

277 DEV_ROOT_PATH = DRBD_BY_RES_PATH 

278 

279 # Default sector size. 

280 BLOCK_SIZE = 512 

281 

282 # List of volume properties. 

283 PROP_METADATA = 'metadata' 

284 PROP_NOT_EXISTS = 'not-exists' 

285 PROP_VOLUME_NAME = 'volume-name' 

286 PROP_IS_READONLY_TIMESTAMP = 'readonly-timestamp' 

287 

288 # A volume can only be locked for a limited duration. 

289 # The goal is to give enough time to slaves to execute some actions on 

290 # a device before an UUID update or a coalesce for example. 

291 # Expiration is expressed in seconds. 

292 LOCKED_EXPIRATION_DELAY = 1 * 60 

293 

294 # Used when volume uuid is being updated. 

295 PROP_UPDATING_UUID_SRC = 'updating-uuid-src' 

296 

297 # States of property PROP_NOT_EXISTS. 

298 STATE_EXISTS = '0' 

299 STATE_NOT_EXISTS = '1' 

300 STATE_CREATING = '2' 

301 

302 # Property namespaces. 

303 NAMESPACE_SR = 'xcp/sr' 

304 NAMESPACE_VOLUME = 'xcp/volume' 

305 

306 # Regex to match properties. 

307 REG_PROP = '^([^/]+)/{}$' 

308 

309 REG_METADATA = re.compile(REG_PROP.format(PROP_METADATA)) 

310 REG_NOT_EXISTS = re.compile(REG_PROP.format(PROP_NOT_EXISTS)) 

311 REG_VOLUME_NAME = re.compile(REG_PROP.format(PROP_VOLUME_NAME)) 

312 REG_UPDATING_UUID_SRC = re.compile(REG_PROP.format(PROP_UPDATING_UUID_SRC)) 

313 

314 # Prefixes of SR/VOLUME in the LINSTOR DB. 

315 # A LINSTOR (resource, group, ...) name cannot start with a number. 

316 # So we add a prefix behind our SR/VOLUME uuids. 

317 PREFIX_SR = 'xcp-sr-' 

318 PREFIX_HA = 'xcp-ha-' 

319 PREFIX_VOLUME = 'xcp-volume-' 

320 

321 # Limit request number when storage pool info is asked, we fetch 

322 # the current pool status after N elapsed seconds. 

323 STORAGE_POOLS_FETCH_INTERVAL = 15 

324 

325 @staticmethod 

326 def default_logger(*args): 

327 print(args) 

328 

329 # -------------------------------------------------------------------------- 

330 # API. 

331 # -------------------------------------------------------------------------- 

332 

333 class VolumeInfo(object): 

334 __slots__ = ( 

335 'name', 

336 'allocated_size', # Allocated size, place count is not used. 

337 'virtual_size', # Total virtual available size of this volume 

338 # (i.e. the user size at creation). 

339 'diskful' # Array of nodes that have a diskful volume. 

340 ) 

341 

342 def __init__(self, name): 

343 self.name = name 

344 self.allocated_size = 0 

345 self.virtual_size = 0 

346 self.diskful = [] 

347 

348 @override 

349 def __repr__(self) -> str: 

350 return 'VolumeInfo("{}", {}, {}, {})'.format( 

351 self.name, self.allocated_size, self.virtual_size, 

352 self.diskful 

353 ) 

354 

355 # -------------------------------------------------------------------------- 

356 

357 def __init__( 

358 self, uri, group_name, repair=False, logger=default_logger.__func__, 

359 attempt_count=30 

360 ): 

361 """ 

362 Create a new LinstorVolumeManager object. 

363 :param str uri: URI to communicate with the LINSTOR controller. 

364 :param str group_name: The SR group name to use. 

365 :param bool repair: If true we try to remove bad volumes due to a crash 

366 or unexpected behavior. 

367 :param function logger: Function to log messages. 

368 :param int attempt_count: Number of attempts to join the controller. 

369 """ 

370 

371 self._uri = uri 

372 self._linstor = self._create_linstor_instance( 

373 uri, attempt_count=attempt_count 

374 ) 

375 

376 

377 mismatched_nodes = [ 

378 node for node in self._linstor.node_list().pop().nodes if node.connection_status == "VERSION_MISMATCH" 

379 ] 

380 

381 if mismatched_nodes: 

382 raise LinstorVolumeManagerError( 

383 "Some linstor nodes are not using the same version. " + 

384 f"Incriminated nodes are: {','.join([node.name for node in mismatched_nodes])}" 

385 ) 

386 

387 self._base_group_name = group_name 

388 

389 # Ensure group exists. 

390 group_name = self.build_group_name(group_name) 

391 groups = self._linstor.resource_group_list_raise([group_name]).resource_groups 

392 if not groups: 

393 raise LinstorVolumeManagerError( 

394 'Unable to find `{}` Linstor SR'.format(group_name) 

395 ) 

396 

397 # Ok. ;) 

398 self._logger = logger 

399 self._redundancy = groups[0].select_filter.place_count 

400 self._group_name = group_name 

401 self._ha_group_name = self._build_ha_group_name(self._base_group_name) 

402 self._volumes = set() 

403 self._storage_pools_time = 0 

404 

405 # To increase performance and limit request count to LINSTOR services, 

406 # we use caches. 

407 self._kv_cache = self._create_kv_cache() 

408 self._resource_cache = None 

409 self._resource_cache_dirty = True 

410 self._volume_info_cache = None 

411 self._volume_info_cache_dirty = True 

412 self._resources_info_cache = None 

413 self._build_volumes(repair=repair) 

414 

415 @property 

416 def uri(self) -> str: 

417 return self._uri 

418 

419 @property 

420 def native_client(self) -> "linstor.Linstor": 

421 return self._linstor 

422 

423 @property 

424 def group_name(self): 

425 """ 

426 Give the used group name. 

427 :return: The group name. 

428 :rtype: str 

429 """ 

430 return self._base_group_name 

431 

432 @property 

433 def redundancy(self): 

434 """ 

435 Give the used redundancy. 

436 :return: The redundancy. 

437 :rtype: int 

438 """ 

439 return self._redundancy 

440 

441 @property 

442 def volumes(self): 

443 """ 

444 Give the volumes uuid set. 

445 :return: The volumes uuid set. 

446 :rtype: set(str) 

447 """ 

448 return self._volumes 

449 

450 @property 

451 def max_volume_size_allowed(self): 

452 """ 

453 Give the max volume size currently available in B. 

454 :return: The current size. 

455 :rtype: int 

456 """ 

457 

458 candidates = self._find_best_size_candidates() 

459 if not candidates: 

460 raise LinstorVolumeManagerError( 

461 'Failed to get max volume size allowed' 

462 ) 

463 

464 size = candidates[0].max_volume_size 

465 if size < 0: 

466 raise LinstorVolumeManagerError( 

467 'Invalid max volume size allowed given: {}'.format(size) 

468 ) 

469 return self.round_down_volume_size(size * 1024) 

470 

471 @property 

472 def physical_size(self): 

473 """ 

474 Give the total physical size of the SR. 

475 :return: The physical size. 

476 :rtype: int 

477 """ 

478 return self._compute_size('total_capacity') 

479 

480 @property 

481 def physical_free_size(self): 

482 """ 

483 Give the total free physical size of the SR. 

484 :return: The physical free size. 

485 :rtype: int 

486 """ 

487 return self._compute_size('free_capacity') 

488 

489 @property 

490 def allocated_volume_size(self): 

491 """ 

492 Give the allocated size for all volumes. The place count is not 

493 used here. When thick lvm is used, the size for one volume should 

494 be equal to the virtual volume size. With thin lvm, the size is equal 

495 or lower to the volume size. 

496 :return: The allocated size of all volumes. 

497 :rtype: int 

498 """ 

499 

500 # Paths: /res_name/vol_number/size 

501 sizes = {} 

502 

503 for resource in self._get_resource_cache().resources: 

504 if resource.name not in sizes: 

505 current = sizes[resource.name] = {} 

506 else: 

507 current = sizes[resource.name] 

508 

509 for volume in resource.volumes: 

510 # We ignore diskless pools of the form "DfltDisklessStorPool". 

511 if volume.storage_pool_name != self._group_name: 

512 continue 

513 

514 allocated_size = max(volume.allocated_size, 0) 

515 current_allocated_size = current.get(volume.number) or -1 

516 if allocated_size > current_allocated_size: 

517 current[volume.number] = allocated_size 

518 

519 total_size = 0 

520 for volumes in sizes.values(): 

521 for size in volumes.values(): 

522 total_size += size 

523 

524 return total_size * 1024 

525 

526 def get_min_physical_size(self): 

527 """ 

528 Give the minimum physical size of the SR. 

529 I.e. the size of the smallest disk + the number of pools. 

530 :return: The physical min size. 

531 :rtype: tuple(int, int) 

532 """ 

533 size = None 

534 pool_count = 0 

535 for pool in self._get_storage_pools(force=True): 

536 space = pool.free_space 

537 if space: 

538 pool_count += 1 

539 current_size = space.total_capacity 

540 if current_size < 0: 

541 raise LinstorVolumeManagerError( 

542 'Failed to get pool total_capacity attr of `{}`' 

543 .format(pool.node_name) 

544 ) 

545 if size is None or current_size < size: 

546 size = current_size 

547 return (pool_count, (size or 0) * 1024) 

548 

549 @property 

550 def metadata(self): 

551 """ 

552 Get the metadata of the SR. 

553 :return: Dictionary that contains metadata. 

554 :rtype: dict(str, dict) 

555 """ 

556 

557 sr_properties = self._get_sr_properties() 

558 metadata = sr_properties.get(self.PROP_METADATA) 

559 if metadata is not None: 

560 metadata = json.loads(metadata) 

561 if isinstance(metadata, dict): 

562 return metadata 

563 raise LinstorVolumeManagerError( 

564 'Expected dictionary in SR metadata: {}'.format( 

565 self._group_name 

566 ) 

567 ) 

568 

569 return {} 

570 

571 @metadata.setter 

572 def metadata(self, metadata): 

573 """ 

574 Set the metadata of the SR. 

575 :param dict metadata: Dictionary that contains metadata. 

576 """ 

577 

578 assert isinstance(metadata, dict) 

579 sr_properties = self._get_sr_properties() 

580 sr_properties[self.PROP_METADATA] = json.dumps(metadata) 

581 

582 @property 

583 def disconnected_hosts(self): 

584 """ 

585 Get the list of disconnected hosts. 

586 :return: Set that contains disconnected hosts. 

587 :rtype: set(str) 

588 """ 

589 

590 disconnected_hosts = set() 

591 for pool in self._get_storage_pools(): 

592 for report in pool.reports: 

593 if report.ret_code & linstor.consts.WARN_NOT_CONNECTED == \ 

594 linstor.consts.WARN_NOT_CONNECTED: 

595 disconnected_hosts.add(pool.node_name) 

596 break 

597 return disconnected_hosts 

598 

599 def check_volume_exists(self, volume_uuid): 

600 """ 

601 Check if a volume exists in the SR. 

602 :return: True if volume exists. 

603 :rtype: bool 

604 """ 

605 return volume_uuid in self._volumes 

606 

607 def create_volume( 

608 self, 

609 volume_uuid, 

610 size, 

611 persistent=True, 

612 volume_name=None, 

613 high_availability=False 

614 ): 

615 """ 

616 Create a new volume on the SR. 

617 :param str volume_uuid: The volume uuid to use. 

618 :param int size: volume size in B. 

619 :param bool persistent: If false the volume will be unavailable 

620 on the next constructor call LinstorSR(...). 

621 :param str volume_name: If set, this name is used in the LINSTOR 

622 database instead of a generated name. 

623 :param bool high_availability: If set, the volume is created in 

624 the HA group. 

625 :return: The current device path of the volume. 

626 :rtype: str 

627 """ 

628 

629 self._logger('Creating LINSTOR volume {}...'.format(volume_uuid)) 

630 if not volume_name: 

631 volume_name = self.build_volume_name(util.gen_uuid()) 

632 volume_properties = self._create_volume_with_properties( 

633 volume_uuid, 

634 volume_name, 

635 size, 

636 True, # place_resources 

637 high_availability 

638 ) 

639 

640 # Volume created! Now try to find the device path. 

641 try: 

642 self._logger( 

643 'Find device path of LINSTOR volume {}...'.format(volume_uuid) 

644 ) 

645 device_path = self._find_device_path(volume_uuid, volume_name) 

646 if persistent: 

647 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_EXISTS 

648 self._volumes.add(volume_uuid) 

649 self._logger( 

650 'LINSTOR volume {} created!'.format(volume_uuid) 

651 ) 

652 return device_path 

653 except Exception: 

654 # There is an issue to find the path. 

655 # At this point the volume has just been created, so force flag can be used. 

656 self._destroy_volume(volume_uuid, force=True) 

657 raise 

658 

659 def mark_volume_as_persistent(self, volume_uuid): 

660 """ 

661 Mark volume as persistent if created with persistent=False. 

662 :param str volume_uuid: The volume uuid to mark. 

663 """ 

664 

665 self._ensure_volume_exists(volume_uuid) 

666 

667 # Mark volume as persistent. 

668 volume_properties = self._get_volume_properties(volume_uuid) 

669 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_EXISTS 

670 

671 def destroy_volume(self, volume_uuid): 

672 """ 

673 Destroy a volume. 

674 :param str volume_uuid: The volume uuid to destroy. 

675 """ 

676 

677 self._ensure_volume_exists(volume_uuid) 

678 self.ensure_volume_is_not_locked(volume_uuid) 

679 

680 is_volume_in_use = any(node["in-use"] for node in self.get_resource_info(volume_uuid)["nodes"].values()) 

681 if is_volume_in_use: 

682 raise LinstorVolumeManagerError( 

683 f"Could not destroy volume `{volume_uuid}` as it is currently in use", 

684 LinstorVolumeManagerError.ERR_VOLUME_IN_USE 

685 ) 

686 

687 # Mark volume as destroyed. 

688 volume_properties = self._get_volume_properties(volume_uuid) 

689 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_NOT_EXISTS 

690 

691 try: 

692 self._volumes.remove(volume_uuid) 

693 self._destroy_volume(volume_uuid) 

694 except Exception as e: 

695 raise LinstorVolumeManagerError( 

696 str(e), 

697 LinstorVolumeManagerError.ERR_VOLUME_DESTROY 

698 ) 

699 

700 def lock_volume(self, volume_uuid, locked=True): 

701 """ 

702 Prevent modifications of the volume properties during 

703 "self.LOCKED_EXPIRATION_DELAY" seconds. The SR must be locked 

704 when used. This method is useful to attach/detach correctly a volume on 

705 a slave. Without it the GC can rename a volume, in this case the old 

706 volume path can be used by a slave... 

707 :param str volume_uuid: The volume uuid to protect/unprotect. 

708 :param bool locked: Lock/unlock the volume. 

709 """ 

710 

711 self._ensure_volume_exists(volume_uuid) 

712 

713 self._logger( 

714 '{} volume {} as locked'.format( 

715 'Mark' if locked else 'Unmark', 

716 volume_uuid 

717 ) 

718 ) 

719 

720 volume_properties = self._get_volume_properties(volume_uuid) 

721 if locked: 

722 volume_properties[ 

723 self.PROP_IS_READONLY_TIMESTAMP 

724 ] = str(time.time()) 

725 elif self.PROP_IS_READONLY_TIMESTAMP in volume_properties: 

726 volume_properties.pop(self.PROP_IS_READONLY_TIMESTAMP) 

727 

728 def ensure_volume_is_not_locked(self, volume_uuid, timeout=None): 

729 """ 

730 Ensure a volume is not locked. Wait if necessary. 

731 :param str volume_uuid: The volume uuid to check. 

732 :param int timeout: If the volume is always locked after the expiration 

733 of the timeout, an exception is thrown. 

734 """ 

735 return self.ensure_volume_list_is_not_locked([volume_uuid], timeout) 

736 

737 def ensure_volume_list_is_not_locked(self, volume_uuids, timeout=None): 

738 checked = set() 

739 for volume_uuid in volume_uuids: 

740 if volume_uuid in self._volumes: 

741 checked.add(volume_uuid) 

742 

743 if not checked: 

744 return 

745 

746 waiting = False 

747 

748 volume_properties = self._get_kv_cache() 

749 

750 start = time.time() 

751 while True: 

752 # Can't delete in for loop, use a copy of the list. 

753 remaining = checked.copy() 

754 for volume_uuid in checked: 

755 volume_properties.namespace = \ 

756 self._build_volume_namespace(volume_uuid) 

757 timestamp = volume_properties.get( 

758 self.PROP_IS_READONLY_TIMESTAMP 

759 ) 

760 if timestamp is None: 

761 remaining.remove(volume_uuid) 

762 continue 

763 

764 now = time.time() 

765 if now - float(timestamp) > self.LOCKED_EXPIRATION_DELAY: 

766 self._logger( 

767 'Remove readonly timestamp on {}'.format(volume_uuid) 

768 ) 

769 volume_properties.pop(self.PROP_IS_READONLY_TIMESTAMP) 

770 remaining.remove(volume_uuid) 

771 continue 

772 

773 if not waiting: 

774 self._logger( 

775 'Volume {} is locked, waiting...'.format(volume_uuid) 

776 ) 

777 waiting = True 

778 break 

779 

780 if not remaining: 

781 break 

782 checked = remaining 

783 

784 if timeout is not None and now - start > timeout: 

785 raise LinstorVolumeManagerError( 

786 'volume `{}` is locked and timeout has been reached' 

787 .format(volume_uuid), 

788 LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS 

789 ) 

790 

791 # We must wait to use the volume. After that we can modify it 

792 # ONLY if the SR is locked to avoid bad reads on the slaves. 

793 time.sleep(1) 

794 volume_properties = self._create_kv_cache() 

795 

796 if waiting: 

797 self._logger('No volume locked now!') 

798 

799 def remove_volume_if_diskless(self, volume_uuid): 

800 """ 

801 Remove disless path from local node. 

802 :param str volume_uuid: The volume uuid to remove. 

803 """ 

804 

805 self._ensure_volume_exists(volume_uuid) 

806 

807 volume_properties = self._get_volume_properties(volume_uuid) 

808 volume_name = volume_properties.get(self.PROP_VOLUME_NAME) 

809 

810 node_name = socket.gethostname() 

811 

812 for resource in self._get_resource_cache().resources: 

813 if resource.name == volume_name and resource.node_name == node_name: 

814 if linstor.consts.FLAG_TIE_BREAKER in resource.flags: 

815 return 

816 break 

817 

818 result = self._linstor.resource_delete_if_diskless( 

819 node_name=node_name, rsc_name=volume_name 

820 ) 

821 if not linstor.Linstor.all_api_responses_no_error(result): 

822 raise LinstorVolumeManagerError( 

823 'Unable to delete diskless path of `{}` on node `{}`: {}' 

824 .format(volume_name, node_name, ', '.join( 

825 [str(x) for x in result])) 

826 ) 

827 

828 def introduce_volume(self, volume_uuid): 

829 pass # TODO: Implement me. 

830 

831 def resize_volume(self, volume_uuid, new_size): 

832 """ 

833 Resize a volume. 

834 :param str volume_uuid: The volume uuid to resize. 

835 :param int new_size: New size in B. 

836 """ 

837 

838 volume_name = self.get_volume_name(volume_uuid) 

839 self.ensure_volume_is_not_locked(volume_uuid) 

840 new_size = self.round_up_volume_size(new_size) // 1024 

841 

842 # We can't resize anything until DRBD is up to date. 

843 # We wait here for 5min max and raise an easy to understand error for the user. 

844 # 5min is an arbitrary time, it's impossible to get a fit all situation value 

845 # and it's currently impossible to know how much time we have to wait 

846 # This is mostly an issue for thick provisioning, thin isn't affected. 

847 start_time = time.monotonic() 

848 try: 

849 self._linstor.resource_dfn_wait_synced(volume_name, wait_interval=1.0, timeout=60*5) 

850 except linstor.LinstorTimeoutError: 

851 raise LinstorVolumeManagerError( 

852 f"Volume resizing of `{volume_uuid}` from SR `{self._group_name}` is incomplete: timeout reached but it continues in background." 

853 ) 

854 util.SMlog(f"DRBD is up to date, syncing took {time.monotonic() - start_time}s") 

855 

856 result = self._linstor.volume_dfn_modify( 

857 rsc_name=volume_name, 

858 volume_nr=0, 

859 size=new_size 

860 ) 

861 

862 self._mark_resource_cache_as_dirty() 

863 

864 error_str = self._get_error_str(result) 

865 if error_str: 

866 raise LinstorVolumeManagerError( 

867 f"Could not resize volume `{volume_uuid}` from SR `{self._group_name}`: {error_str}" 

868 ) 

869 

870 def get_volume_name(self, volume_uuid): 

871 """ 

872 Get the name of a particular volume. 

873 :param str volume_uuid: The volume uuid of the name to get. 

874 :return: The volume name. 

875 :rtype: str 

876 """ 

877 

878 self._ensure_volume_exists(volume_uuid) 

879 volume_properties = self._get_volume_properties(volume_uuid) 

880 volume_name = volume_properties.get(self.PROP_VOLUME_NAME) 

881 if volume_name: 

882 return volume_name 

883 raise LinstorVolumeManagerError( 

884 'Failed to get volume name of {}'.format(volume_uuid) 

885 ) 

886 

887 def get_volume_size(self, volume_uuid): 

888 """ 

889 Get the size of a particular volume. 

890 :param str volume_uuid: The volume uuid of the size to get. 

891 :return: The volume size. 

892 :rtype: int 

893 """ 

894 

895 volume_name = self.get_volume_name(volume_uuid) 

896 dfns = self._linstor.resource_dfn_list_raise( 

897 query_volume_definitions=True, 

898 filter_by_resource_definitions=[volume_name] 

899 ).resource_definitions 

900 

901 size = dfns[0].volume_definitions[0].size 

902 if size < 0: 

903 raise LinstorVolumeManagerError( 

904 'Failed to get volume size of: {}'.format(volume_uuid) 

905 ) 

906 return size * 1024 

907 

908 def set_auto_promote_timeout(self, volume_uuid, timeout): 

909 """ 

910 Define the blocking time of open calls when a DRBD 

911 is already open on another host. 

912 :param str volume_uuid: The volume uuid to modify. 

913 """ 

914 

915 volume_name = self.get_volume_name(volume_uuid) 

916 result = self._linstor.resource_dfn_modify(volume_name, { 

917 'DrbdOptions/Resource/auto-promote-timeout': timeout 

918 }) 

919 error_str = self._get_error_str(result) 

920 if error_str: 

921 raise LinstorVolumeManagerError( 

922 'Could not change the auto promote timeout of `{}`: {}' 

923 .format(volume_uuid, error_str) 

924 ) 

925 

926 def set_drbd_ha_properties(self, volume_name, enabled=True): 

927 """ 

928 Set or not HA DRBD properties required by drbd-reactor and 

929 by specific volumes. 

930 :param str volume_name: The volume to modify. 

931 :param bool enabled: Enable or disable HA properties. 

932 """ 

933 

934 properties = { 

935 'DrbdOptions/auto-quorum': 'disabled', 

936 'DrbdOptions/Resource/auto-promote': 'no', 

937 'DrbdOptions/Resource/on-no-data-accessible': 'io-error', 

938 'DrbdOptions/Resource/on-no-quorum': 'io-error', 

939 'DrbdOptions/Resource/on-suspended-primary-outdated': 'force-secondary', 

940 'DrbdOptions/Resource/quorum': 'majority' 

941 } 

942 if enabled: 

943 result = self._linstor.resource_dfn_modify(volume_name, properties) 

944 else: 

945 result = self._linstor.resource_dfn_modify(volume_name, {}, delete_props=list(properties.keys())) 

946 

947 error_str = self._get_error_str(result) 

948 if error_str: 

949 raise LinstorVolumeManagerError( 

950 'Could not modify HA DRBD properties on volume `{}`: {}' 

951 .format(volume_name, error_str) 

952 ) 

953 

954 def get_volume_info(self, volume_uuid): 

955 """ 

956 Get the volume info of a particular volume. 

957 :param str volume_uuid: The volume uuid of the volume info to get. 

958 :return: The volume info. 

959 :rtype: VolumeInfo 

960 """ 

961 

962 volume_name = self.get_volume_name(volume_uuid) 

963 return self._get_volumes_info()[volume_name] 

964 

965 def get_device_path(self, volume_uuid): 

966 """ 

967 Get the dev path of a volume, create a diskless if necessary. 

968 :param str volume_uuid: The volume uuid to get the dev path. 

969 :return: The current device path of the volume. 

970 :rtype: str 

971 """ 

972 

973 volume_name = self.get_volume_name(volume_uuid) 

974 return self._find_device_path(volume_uuid, volume_name) 

975 

976 def get_volume_uuid_from_device_path(self, device_path): 

977 """ 

978 Get the volume uuid of a device_path. 

979 :param str device_path: The dev path to find the volume uuid. 

980 :return: The volume uuid of the local device path. 

981 :rtype: str 

982 """ 

983 

984 expected_volume_name = \ 

985 self.get_volume_name_from_device_path(device_path) 

986 

987 volume_names = self.get_volumes_with_name() 

988 for volume_uuid, volume_name in volume_names.items(): 

989 if volume_name == expected_volume_name: 

990 return volume_uuid 

991 

992 raise LinstorVolumeManagerError( 

993 'Unable to find volume uuid from dev path `{}`'.format(device_path) 

994 ) 

995 

996 def get_volume_name_from_device_path(self, device_path): 

997 """ 

998 Get the volume name of a device_path. 

999 :param str device_path: The dev path to find the volume name. 

1000 :return: The volume name of the device path. 

1001 :rtype: str 

1002 """ 

1003 

1004 # Assume that we have a path like this: 

1005 # - "/dev/drbd/by-res/xcp-volume-<UUID>/0" 

1006 # - "../xcp-volume-<UUID>/0" 

1007 if device_path.startswith(DRBD_BY_RES_PATH): 

1008 prefix_len = len(DRBD_BY_RES_PATH) 

1009 elif device_path.startswith('../'): 

1010 prefix_len = 3 

1011 else: 

1012 raise LinstorVolumeManagerError('Unexpected device path: `{}`'.format(device_path)) 

1013 

1014 res_name_end = device_path.find('/', prefix_len) 

1015 assert res_name_end != -1 

1016 return device_path[prefix_len:res_name_end] 

1017 

1018 def update_volume_uuid(self, volume_uuid, new_volume_uuid, force=False): 

1019 """ 

1020 Change the uuid of a volume. 

1021 :param str volume_uuid: The volume to modify. 

1022 :param str new_volume_uuid: The new volume uuid to use. 

1023 :param bool force: If true we doesn't check if volume_uuid is in the 

1024 volume list. I.e. the volume can be marked as deleted but the volume 

1025 can still be in the LINSTOR KV store if the deletion has failed. 

1026 In specific cases like "undo" after a failed clone we must rename a bad 

1027 deleted VDI. 

1028 """ 

1029 

1030 self._logger( 

1031 'Trying to update volume UUID {} to {}...' 

1032 .format(volume_uuid, new_volume_uuid) 

1033 ) 

1034 assert volume_uuid != new_volume_uuid, 'can\'t update volume UUID, same value' 

1035 

1036 if not force: 

1037 self._ensure_volume_exists(volume_uuid) 

1038 self.ensure_volume_is_not_locked(volume_uuid) 

1039 

1040 if new_volume_uuid in self._volumes: 

1041 raise LinstorVolumeManagerError( 

1042 'Volume `{}` already exists'.format(new_volume_uuid), 

1043 LinstorVolumeManagerError.ERR_VOLUME_EXISTS 

1044 ) 

1045 

1046 volume_properties = self._get_volume_properties(volume_uuid) 

1047 if volume_properties.get(self.PROP_UPDATING_UUID_SRC): 

1048 raise LinstorVolumeManagerError( 

1049 'Cannot update volume uuid {}: invalid state' 

1050 .format(volume_uuid) 

1051 ) 

1052 

1053 # 1. Copy in temp variables metadata and volume_name. 

1054 metadata = volume_properties.get(self.PROP_METADATA) 

1055 volume_name = volume_properties.get(self.PROP_VOLUME_NAME) 

1056 

1057 # 2. Switch to new volume namespace. 

1058 volume_properties.namespace = self._build_volume_namespace( 

1059 new_volume_uuid 

1060 ) 

1061 

1062 if list(volume_properties.items()): 

1063 raise LinstorVolumeManagerError( 

1064 'Cannot update volume uuid {} to {}: ' 

1065 .format(volume_uuid, new_volume_uuid) + 

1066 'this last one is not empty' 

1067 ) 

1068 

1069 try: 

1070 # 3. Mark new volume properties with PROP_UPDATING_UUID_SRC. 

1071 # If we crash after that, the new properties can be removed 

1072 # properly. 

1073 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_NOT_EXISTS 

1074 volume_properties[self.PROP_UPDATING_UUID_SRC] = volume_uuid 

1075 

1076 # 4. Copy the properties. 

1077 # Note: On new volumes, during clone for example, the metadata 

1078 # may be missing. So we must test it to avoid this error: 

1079 # "None has to be a str/unicode, but is <type 'NoneType'>" 

1080 if metadata: 

1081 volume_properties[self.PROP_METADATA] = metadata 

1082 volume_properties[self.PROP_VOLUME_NAME] = volume_name 

1083 

1084 # 5. Ok! 

1085 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_EXISTS 

1086 except Exception as err: 

1087 try: 

1088 # Clear the new volume properties in case of failure. 

1089 assert volume_properties.namespace == \ 

1090 self._build_volume_namespace(new_volume_uuid) 

1091 volume_properties.clear() 

1092 except Exception as e: 

1093 self._logger( 

1094 'Failed to clear new volume properties: {} (ignoring...)' 

1095 .format(e) 

1096 ) 

1097 raise LinstorVolumeManagerError( 

1098 'Failed to copy volume properties: {}'.format(err) 

1099 ) 

1100 

1101 try: 

1102 # 6. After this point, it's ok we can remove the 

1103 # PROP_UPDATING_UUID_SRC property and clear the src properties 

1104 # without problems. 

1105 

1106 # 7. Switch to old volume namespace. 

1107 volume_properties.namespace = self._build_volume_namespace( 

1108 volume_uuid 

1109 ) 

1110 volume_properties.clear() 

1111 

1112 # 8. Switch a last time to new volume namespace. 

1113 volume_properties.namespace = self._build_volume_namespace( 

1114 new_volume_uuid 

1115 ) 

1116 volume_properties.pop(self.PROP_UPDATING_UUID_SRC) 

1117 except Exception as e: 

1118 raise LinstorVolumeManagerError( 

1119 'Failed to clear volume properties ' 

1120 'after volume uuid update: {}'.format(e) 

1121 ) 

1122 

1123 try: 

1124 self._volumes.remove(volume_uuid) 

1125 except KeyError: 

1126 # Can be missing if we are building the volume set attr AND 

1127 # we are processing a deleted resource. 

1128 assert force 

1129 

1130 self._volumes.add(new_volume_uuid) 

1131 

1132 self._logger( 

1133 'UUID update succeeded of {} to {}! (properties={})' 

1134 .format( 

1135 volume_uuid, new_volume_uuid, 

1136 self._get_filtered_properties(volume_properties) 

1137 ) 

1138 ) 

1139 

1140 def update_volume_name(self, volume_uuid, volume_name): 

1141 """ 

1142 Change the volume name of a volume. 

1143 :param str volume_uuid: The volume to modify. 

1144 :param str volume_name: The volume_name to use. 

1145 """ 

1146 

1147 self._ensure_volume_exists(volume_uuid) 

1148 self.ensure_volume_is_not_locked(volume_uuid) 

1149 if not volume_name.startswith(self.PREFIX_VOLUME): 

1150 raise LinstorVolumeManagerError( 

1151 'Volume name `{}` must be start with `{}`' 

1152 .format(volume_name, self.PREFIX_VOLUME) 

1153 ) 

1154 

1155 if volume_name not in self._fetch_resource_names(): 

1156 raise LinstorVolumeManagerError( 

1157 'Volume `{}` doesn\'t exist'.format(volume_name) 

1158 ) 

1159 

1160 volume_properties = self._get_volume_properties(volume_uuid) 

1161 volume_properties[self.PROP_VOLUME_NAME] = volume_name 

1162 

1163 def get_usage_states(self, volume_uuid): 

1164 """ 

1165 Check if a volume is currently used. 

1166 :param str volume_uuid: The volume uuid to check. 

1167 :return: A dictionary that contains states. 

1168 :rtype: dict(str, bool or None) 

1169 """ 

1170 

1171 states = {} 

1172 

1173 volume_name = self.get_volume_name(volume_uuid) 

1174 for resource_state in self._linstor.resource_list_raise( 

1175 filter_by_resources=[volume_name] 

1176 ).resource_states: 

1177 states[resource_state.node_name] = resource_state.in_use 

1178 

1179 return states 

1180 

1181 def get_volume_openers(self, volume_uuid) -> LinstorVolumeOpeners: 

1182 """ 

1183 Get openers of a volume. 

1184 :param str volume_uuid: The volume uuid to monitor. 

1185 :return: A dictionary that contains openers. 

1186 :rtype: dict(str, obj) 

1187 """ 

1188 return get_all_volume_openers(self.get_volume_name(volume_uuid), '0') 

1189 

1190 def get_volumes_with_name(self): 

1191 """ 

1192 Give a volume dictionary that contains names actually owned. 

1193 :return: A volume/name dict. 

1194 :rtype: dict(str, str) 

1195 """ 

1196 return self._get_volumes_by_property(self.REG_VOLUME_NAME) 

1197 

1198 def get_volumes_with_info(self): 

1199 """ 

1200 Give a volume dictionary that contains VolumeInfos. 

1201 :return: A volume/VolumeInfo dict. 

1202 :rtype: dict(str, VolumeInfo) 

1203 """ 

1204 

1205 volumes = {} 

1206 

1207 volume_names = self.get_volumes_with_name() 

1208 all_volume_info = self._get_volumes_info(volume_names) 

1209 for volume_uuid, volume_name in volume_names.items(): 

1210 if volume_name: 

1211 volume_info = all_volume_info.get(volume_name) 

1212 if volume_info: 

1213 volumes[volume_uuid] = volume_info 

1214 continue 

1215 

1216 # Well I suppose if this volume is not available, 

1217 # LINSTOR has been used directly without using this API. 

1218 volumes[volume_uuid] = self.VolumeInfo('') 

1219 

1220 return volumes 

1221 

1222 def get_volumes_with_metadata(self): 

1223 """ 

1224 Give a volume dictionary that contains metadata. 

1225 :return: A volume/metadata dict. 

1226 :rtype: dict(str, dict) 

1227 """ 

1228 

1229 volumes = {} 

1230 

1231 metadata = self._get_volumes_by_property(self.REG_METADATA) 

1232 for volume_uuid, volume_metadata in metadata.items(): 

1233 if volume_metadata: 

1234 volume_metadata = json.loads(volume_metadata) 

1235 if isinstance(volume_metadata, dict): 

1236 volumes[volume_uuid] = volume_metadata 

1237 continue 

1238 raise LinstorVolumeManagerError( 

1239 'Expected dictionary in volume metadata: {}' 

1240 .format(volume_uuid) 

1241 ) 

1242 

1243 volumes[volume_uuid] = {} 

1244 

1245 return volumes 

1246 

1247 def get_volume_metadata(self, volume_uuid): 

1248 """ 

1249 Get the metadata of a volume. 

1250 :return: Dictionary that contains metadata. 

1251 :rtype: dict 

1252 """ 

1253 

1254 self._ensure_volume_exists(volume_uuid) 

1255 volume_properties = self._get_volume_properties(volume_uuid) 

1256 metadata = volume_properties.get(self.PROP_METADATA) 

1257 if metadata: 

1258 metadata = json.loads(metadata) 

1259 if isinstance(metadata, dict): 

1260 return metadata 

1261 raise LinstorVolumeManagerError( 

1262 'Expected dictionary in volume metadata: {}' 

1263 .format(volume_uuid) 

1264 ) 

1265 return {} 

1266 

1267 def set_volume_metadata(self, volume_uuid, metadata): 

1268 """ 

1269 Set the metadata of a volume. 

1270 :param dict metadata: Dictionary that contains metadata. 

1271 """ 

1272 

1273 self._ensure_volume_exists(volume_uuid) 

1274 self.ensure_volume_is_not_locked(volume_uuid) 

1275 

1276 assert isinstance(metadata, dict) 

1277 volume_properties = self._get_volume_properties(volume_uuid) 

1278 volume_properties[self.PROP_METADATA] = json.dumps(metadata) 

1279 

1280 def update_volume_metadata(self, volume_uuid, metadata): 

1281 """ 

1282 Update the metadata of a volume. It modify only the given keys. 

1283 It doesn't remove unreferenced key instead of set_volume_metadata. 

1284 :param dict metadata: Dictionary that contains metadata. 

1285 """ 

1286 

1287 self._ensure_volume_exists(volume_uuid) 

1288 self.ensure_volume_is_not_locked(volume_uuid) 

1289 

1290 assert isinstance(metadata, dict) 

1291 volume_properties = self._get_volume_properties(volume_uuid) 

1292 

1293 current_metadata = json.loads( 

1294 volume_properties.get(self.PROP_METADATA, '{}') 

1295 ) 

1296 if not isinstance(metadata, dict): 

1297 raise LinstorVolumeManagerError( 

1298 'Expected dictionary in volume metadata: {}' 

1299 .format(volume_uuid) 

1300 ) 

1301 

1302 for key, value in metadata.items(): 

1303 current_metadata[key] = value 

1304 volume_properties[self.PROP_METADATA] = json.dumps(current_metadata) 

1305 

1306 def shallow_clone_volume(self, volume_uuid, clone_uuid, persistent=True): 

1307 """ 

1308 Clone a volume. Do not copy the data, this method creates a new volume 

1309 with the same size. 

1310 :param str volume_uuid: The volume to clone. 

1311 :param str clone_uuid: The cloned volume. 

1312 :param bool persistent: If false the volume will be unavailable 

1313 on the next constructor call LinstorSR(...). 

1314 :return: The current device path of the cloned volume. 

1315 :rtype: str 

1316 """ 

1317 

1318 volume_name = self.get_volume_name(volume_uuid) 

1319 self.ensure_volume_is_not_locked(volume_uuid) 

1320 

1321 # 1. Find ideal nodes + size to use. 

1322 ideal_node_names, size = self._get_volume_node_names_and_size( 

1323 volume_name 

1324 ) 

1325 if size <= 0: 

1326 raise LinstorVolumeManagerError( 

1327 'Invalid size of {} for volume `{}`'.format(size, volume_name) 

1328 ) 

1329 

1330 # 2. Create clone! 

1331 return self.create_volume(clone_uuid, size, persistent) 

1332 

1333 def remove_resourceless_volumes(self): 

1334 """ 

1335 Remove all volumes without valid or non-empty name 

1336 (i.e. without LINSTOR resource). It's different than 

1337 LinstorVolumeManager constructor that takes a `repair` param that 

1338 removes volumes with `PROP_NOT_EXISTS` to 1. 

1339 """ 

1340 

1341 resource_names = self._fetch_resource_names() 

1342 for volume_uuid, volume_name in self.get_volumes_with_name().items(): 

1343 if not volume_name or volume_name not in resource_names: 

1344 # Don't force, we can be sure of what's happening. 

1345 self.destroy_volume(volume_uuid) 

1346 

1347 def destroy(self): 

1348 """ 

1349 Destroy this SR. Object should not be used after that. 

1350 :param bool force: Try to destroy volumes before if true. 

1351 """ 

1352 

1353 # 1. Ensure volume list is empty. No cost. 

1354 if self._volumes: 

1355 raise LinstorVolumeManagerError( 

1356 'Cannot destroy LINSTOR volume manager: ' 

1357 'It exists remaining volumes' 

1358 ) 

1359 

1360 # 2. Fetch ALL resource names. 

1361 # This list may therefore contain volumes created outside 

1362 # the scope of the driver. 

1363 resource_names = self._fetch_resource_names(ignore_deleted=False) 

1364 try: 

1365 resource_names.remove(DATABASE_VOLUME_NAME) 

1366 except KeyError: 

1367 # Really strange to reach that point. 

1368 # Normally we always have the database volume in the list. 

1369 pass 

1370 

1371 # 3. Ensure the resource name list is entirely empty... 

1372 if resource_names: 

1373 raise LinstorVolumeManagerError( 

1374 'Cannot destroy LINSTOR volume manager: ' 

1375 'It exists remaining volumes (created externally or being deleted)' 

1376 ) 

1377 

1378 # 4. Destroying... 

1379 controller_is_running = self._controller_is_running() 

1380 uri = 'linstor://localhost' 

1381 try: 

1382 if controller_is_running: 

1383 self._start_controller(start=False) 

1384 

1385 # 4.1. Umount LINSTOR database. 

1386 self._mount_database_volume( 

1387 self.build_device_path(DATABASE_VOLUME_NAME), 

1388 mount=False, 

1389 force=True 

1390 ) 

1391 

1392 # 4.2. Refresh instance. 

1393 self._start_controller(start=True) 

1394 self._linstor = self._create_linstor_instance( 

1395 uri, keep_uri_unmodified=True 

1396 ) 

1397 

1398 # 4.3. Destroy database volume. 

1399 self._destroy_resource(DATABASE_VOLUME_NAME) 

1400 

1401 # 4.4. Refresh linstor connection. 

1402 # Without we get this error: 

1403 # "Cannot delete resource group 'xcp-sr-linstor_group_thin_device' because it has existing resource definitions.." 

1404 # Because the deletion of the databse was not seen by Linstor for some reason. 

1405 # It seems a simple refresh of the Linstor connection make it aware of the deletion. 

1406 self._linstor.disconnect() 

1407 self._linstor.connect() 

1408 

1409 # 4.5. Destroy remaining drbd nodes on hosts. 

1410 # We check if there is a DRBD node on hosts that could mean blocking when destroying resource groups. 

1411 # It needs to be done locally by each host so we go through the linstor-manager plugin. 

1412 # If we don't do this sometimes, the destroy will fail when trying to destroy the resource groups with: 

1413 # "linstor-manager:destroy error: Failed to destroy SP `xcp-sr-linstor_group_thin_device` on node `r620-s2`: The specified storage pool 'xcp-sr-linstor_group_thin_device' on node 'r620-s2' can not be deleted as volumes / snapshot-volumes are still using it." 

1414 session = util.timeout(5, util.get_localAPI_session) 

1415 for host_ref in session.xenapi.host.get_all(): 

1416 try: 

1417 response = session.xenapi.host.call_plugin( 

1418 host_ref, 'linstor-manager', 'destroyDrbdVolumes', {'volume_group': self._group_name} 

1419 ) 

1420 except Exception as e: 

1421 util.SMlog('Calling destroyDrbdVolumes on host {} failed with error {}'.format(host_ref, e)) 

1422 

1423 # 4.6. Destroy group and storage pools. 

1424 self._destroy_resource_group(self._linstor, self._group_name) 

1425 self._destroy_resource_group(self._linstor, self._ha_group_name) 

1426 for pool in self._get_storage_pools(force=True): 

1427 self._destroy_storage_pool( 

1428 self._linstor, pool.name, pool.node_name 

1429 ) 

1430 except Exception as e: 

1431 self._start_controller(start=controller_is_running) 

1432 raise e 

1433 

1434 try: 

1435 self._start_controller(start=False) 

1436 for file in os.listdir(DATABASE_PATH): 

1437 if file != 'lost+found': 

1438 os.remove(DATABASE_PATH + '/' + file) 

1439 except Exception as e: 

1440 util.SMlog( 

1441 'Ignoring failure after LINSTOR SR destruction: {}' 

1442 .format(e) 

1443 ) 

1444 

1445 def find_up_to_date_diskful_nodes(self, volume_uuid): 

1446 """ 

1447 Find all nodes that contain a specific volume using diskful disks. 

1448 The disk must be up to data to be used. 

1449 :param str volume_uuid: The volume to use. 

1450 :return: The available nodes. 

1451 :rtype: tuple(set(str), str) 

1452 """ 

1453 

1454 volume_name = self.get_volume_name(volume_uuid) 

1455 

1456 in_use_by = None 

1457 node_names = set() 

1458 

1459 resource_states = filter( 

1460 lambda resource_state: resource_state.name == volume_name, 

1461 self._get_resource_cache().resource_states 

1462 ) 

1463 

1464 for resource_state in resource_states: 

1465 volume_state = resource_state.volume_states[0] 

1466 if volume_state.disk_state == 'UpToDate': 

1467 node_names.add(resource_state.node_name) 

1468 if resource_state.in_use: 

1469 in_use_by = resource_state.node_name 

1470 

1471 return (node_names, in_use_by) 

1472 

1473 def invalidate_resource_cache(self): 

1474 """ 

1475 If resources are impacted by external commands like vhdutil, 

1476 it's necessary to call this function to invalidate current resource 

1477 cache. 

1478 """ 

1479 self._mark_resource_cache_as_dirty() 

1480 

1481 def has_node(self, node_name): 

1482 """ 

1483 Check if a node exists in the LINSTOR database. 

1484 :rtype: bool 

1485 """ 

1486 result = self._linstor.node_list() 

1487 error_str = self._get_error_str(result) 

1488 if error_str: 

1489 raise LinstorVolumeManagerError( 

1490 'Failed to list nodes using `{}`: {}' 

1491 .format(node_name, error_str) 

1492 ) 

1493 return bool(result[0].node(node_name)) 

1494 

1495 def create_node(self, node_name, ip): 

1496 """ 

1497 Create a new node in the LINSTOR database. 

1498 :param str node_name: Node name to use. 

1499 :param str ip: Host IP to communicate. 

1500 """ 

1501 result = self._linstor.node_create( 

1502 node_name, 

1503 linstor.consts.VAL_NODE_TYPE_CMBD, 

1504 ip 

1505 ) 

1506 errors = self._filter_errors(result) 

1507 if errors: 

1508 error_str = self._get_error_str(errors) 

1509 raise LinstorVolumeManagerError( 

1510 'Failed to create node `{}`: {}'.format(node_name, error_str) 

1511 ) 

1512 

1513 def destroy_node(self, node_name): 

1514 """ 

1515 Destroy a node in the LINSTOR database. 

1516 :param str node_name: Node name to remove. 

1517 """ 

1518 result = self._linstor.node_delete(node_name) 

1519 errors = self._filter_errors(result) 

1520 if errors: 

1521 error_str = self._get_error_str(errors) 

1522 raise LinstorVolumeManagerError( 

1523 'Failed to destroy node `{}`: {}'.format(node_name, error_str) 

1524 ) 

1525 

1526 def create_node_interface(self, node_name, name, ip): 

1527 """ 

1528 Create a new node interface in the LINSTOR database. 

1529 :param str node_name: Node name of the interface to use. 

1530 :param str name: Interface to create. 

1531 :param str ip: IP of the interface. 

1532 """ 

1533 result = self._linstor.netinterface_create(node_name, name, ip) 

1534 errors = self._filter_errors(result) 

1535 if errors: 

1536 error_str = self._get_error_str(errors) 

1537 raise LinstorVolumeManagerError( 

1538 'Failed to create node interface on `{}`: {}'.format(node_name, error_str) 

1539 ) 

1540 

1541 def destroy_node_interface(self, node_name, name): 

1542 """ 

1543 Destroy a node interface in the LINSTOR database. 

1544 :param str node_name: Node name of the interface to remove. 

1545 :param str name: Interface to remove. 

1546 """ 

1547 

1548 if name == 'default': 

1549 raise LinstorVolumeManagerError( 

1550 'Unable to delete the default interface of a node!' 

1551 ) 

1552 

1553 result = self._linstor.netinterface_delete(node_name, name) 

1554 errors = self._filter_errors(result) 

1555 if errors: 

1556 error_str = self._get_error_str(errors) 

1557 raise LinstorVolumeManagerError( 

1558 'Failed to destroy node interface on `{}`: {}'.format(node_name, error_str) 

1559 ) 

1560 

1561 def modify_node_interface(self, node_name, name, ip): 

1562 """ 

1563 Modify a node interface in the LINSTOR database. Create it if necessary. 

1564 :param str node_name: Node name of the interface to use. 

1565 :param str name: Interface to modify or create. 

1566 :param str ip: IP of the interface. 

1567 """ 

1568 result = self._linstor.netinterface_create(node_name, name, ip) 

1569 errors = self._filter_errors(result) 

1570 if not errors: 

1571 return 

1572 

1573 if self._check_errors(errors, [linstor.consts.FAIL_EXISTS_NET_IF]): 

1574 result = self._linstor.netinterface_modify(node_name, name, ip) 

1575 errors = self._filter_errors(result) 

1576 if not errors: 

1577 return 

1578 

1579 error_str = self._get_error_str(errors) 

1580 raise LinstorVolumeManagerError( 

1581 'Unable to modify interface on `{}`: {}'.format(node_name, error_str) 

1582 ) 

1583 

1584 def list_node_interfaces(self, node_name): 

1585 """ 

1586 List all node interfaces. 

1587 :param str node_name: Node name to use to list interfaces. 

1588 :rtype: list 

1589 : 

1590 """ 

1591 result = self._linstor.net_interface_list(node_name) 

1592 if not result: 

1593 raise LinstorVolumeManagerError( 

1594 'Unable to list interfaces on `{}`: no list received'.format(node_name) 

1595 ) 

1596 

1597 interfaces = {} 

1598 for interface in result: 

1599 interface = interface._rest_data 

1600 interfaces[interface['name']] = { 

1601 'address': interface['address'], 

1602 'active': interface['is_active'] 

1603 } 

1604 return interfaces 

1605 

1606 def get_node_preferred_interface(self, node_name): 

1607 """ 

1608 Get the preferred interface used by a node. 

1609 :param str node_name: Node name of the interface to get. 

1610 :rtype: str 

1611 """ 

1612 try: 

1613 nodes = self._linstor.node_list_raise([node_name]).nodes 

1614 if nodes: 

1615 properties = nodes[0].props 

1616 return properties.get('PrefNic', 'default') 

1617 return nodes 

1618 except Exception as e: 

1619 raise LinstorVolumeManagerError( 

1620 'Failed to get preferred interface: `{}`'.format(e) 

1621 ) 

1622 

1623 def set_node_preferred_interface(self, node_name, name): 

1624 """ 

1625 Set the preferred interface to use on a node. 

1626 :param str node_name: Node name of the interface. 

1627 :param str name: Preferred interface to use. 

1628 """ 

1629 result = self._linstor.node_modify(node_name, property_dict={'PrefNic': name}) 

1630 errors = self._filter_errors(result) 

1631 if errors: 

1632 error_str = self._get_error_str(errors) 

1633 raise LinstorVolumeManagerError( 

1634 'Failed to set preferred node interface on `{}`: {}'.format(node_name, error_str) 

1635 ) 

1636 

1637 def get_nodes_info(self): 

1638 """ 

1639 Get all nodes + statuses, used or not by the pool. 

1640 :rtype: dict(str, dict) 

1641 """ 

1642 try: 

1643 nodes = {} 

1644 for node in self._linstor.node_list_raise().nodes: 

1645 nodes[node.name] = node.connection_status 

1646 return nodes 

1647 except Exception as e: 

1648 raise LinstorVolumeManagerError( 

1649 'Failed to get all nodes: `{}`'.format(e) 

1650 ) 

1651 

1652 def get_storage_pools_info(self): 

1653 """ 

1654 Give all storage pools of current group name. 

1655 :rtype: dict(str, list) 

1656 """ 

1657 storage_pools = {} 

1658 for pool in self._get_storage_pools(force=True): 

1659 if pool.node_name not in storage_pools: 

1660 storage_pools[pool.node_name] = [] 

1661 

1662 size = -1 

1663 capacity = -1 

1664 

1665 space = pool.free_space 

1666 if space: 

1667 size = space.free_capacity 

1668 if size < 0: 

1669 size = -1 

1670 else: 

1671 size *= 1024 

1672 capacity = space.total_capacity 

1673 if capacity <= 0: 

1674 capacity = -1 

1675 else: 

1676 capacity *= 1024 

1677 

1678 storage_pools[pool.node_name].append({ 

1679 'name': pool.name, 

1680 'linstor-uuid': pool.uuid, 

1681 'free-size': size, 

1682 'capacity': capacity 

1683 }) 

1684 

1685 return storage_pools 

1686 

1687 def get_resources_info(self): 

1688 """ 

1689 Give all resources of current group name. 

1690 :rtype: dict(str, list) 

1691 """ 

1692 if self._resources_info_cache and not self._resource_cache_dirty: 

1693 return self._resources_info_cache 

1694 

1695 resources = {} 

1696 resource_list = self._get_resource_cache() 

1697 volume_names = self.get_volumes_with_name() 

1698 for resource in resource_list.resources: 

1699 if resource.name not in resources: 

1700 resources[resource.name] = { 'nodes': {}, 'uuid': '' } 

1701 resource_nodes = resources[resource.name]['nodes'] 

1702 

1703 resource_nodes[resource.node_name] = { 

1704 'volumes': [], 

1705 'diskful': linstor.consts.FLAG_DISKLESS not in resource.flags, 

1706 'tie-breaker': linstor.consts.FLAG_TIE_BREAKER in resource.flags 

1707 } 

1708 resource_volumes = resource_nodes[resource.node_name]['volumes'] 

1709 

1710 for volume in resource.volumes: 

1711 # We ignore diskless pools of the form "DfltDisklessStorPool". 

1712 if volume.storage_pool_name != self._group_name: 

1713 continue 

1714 

1715 usable_size = volume.usable_size 

1716 if usable_size < 0: 

1717 usable_size = -1 

1718 else: 

1719 usable_size *= 1024 

1720 

1721 allocated_size = volume.allocated_size 

1722 if allocated_size < 0: 

1723 allocated_size = -1 

1724 else: 

1725 allocated_size *= 1024 

1726 

1727 resource_volumes.append({ 

1728 'storage-pool-name': volume.storage_pool_name, 

1729 'linstor-uuid': volume.uuid, 

1730 'number': volume.number, 

1731 'device-path': volume.device_path, 

1732 'usable-size': usable_size, 

1733 'allocated-size': allocated_size 

1734 }) 

1735 

1736 for resource_state in resource_list.resource_states: 

1737 resource = resources[resource_state.rsc_name]['nodes'][resource_state.node_name] 

1738 resource['in-use'] = resource_state.in_use 

1739 

1740 volumes = resource['volumes'] 

1741 for volume_state in resource_state.volume_states: 

1742 volume = next((x for x in volumes if x['number'] == volume_state.number), None) 

1743 if volume: 

1744 volume['disk-state'] = volume_state.disk_state 

1745 

1746 for volume_uuid, volume_name in volume_names.items(): 

1747 resource = resources.get(volume_name) 

1748 if resource: 

1749 resource['uuid'] = volume_uuid 

1750 

1751 self._resources_info_cache = resources 

1752 return self._resources_info_cache 

1753 

1754 def get_resource_info(self, volume_uuid: str) -> Dict[str, Any]: 

1755 """ 

1756 Give a resource info based on its UUID. 

1757 :param volume_uuid str: volume uuid to search for 

1758 :rtype: dict(str, any) 

1759 """ 

1760 for volume in self.get_resources_info().values(): 

1761 if volume["uuid"] == volume_uuid: 

1762 return volume 

1763 

1764 raise LinstorVolumeManagerError( 

1765 f"Could not find info about volume `{volume_uuid}`", 

1766 LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS 

1767 ) 

1768 

1769 def get_database_path(self): 

1770 """ 

1771 Get the database path. 

1772 :return: The current database path. 

1773 :rtype: str 

1774 """ 

1775 return self._request_database_path(self._linstor, activate=True) 

1776 

1777 @classmethod 

1778 def get_all_group_names(cls, base_name): 

1779 """ 

1780 Get all group names. I.e. list of current group + HA. 

1781 :param str base_name: The SR group_name to use. 

1782 :return: List of group names. 

1783 :rtype: list 

1784 """ 

1785 return [cls.build_group_name(base_name), cls._build_ha_group_name(base_name)] 

1786 

1787 @classmethod 

1788 def get_volume_group_name(cls, volume_name) -> str: 

1789 """ 

1790 Get the group name associated with a volume. 

1791 :param str volume_name: The volume name to find. 

1792 :return: A group name. 

1793 :rtype: str 

1794 """ 

1795 

1796 lin = cls._create_linstor_instance(uri=None) 

1797 try: 

1798 for dfn in lin.resource_dfn_list_raise( 

1799 query_volume_definitions=False, 

1800 filter_by_resource_definitions=[volume_name] 

1801 ).resource_definitions: 

1802 return dfn.resource_group_name 

1803 finally: 

1804 lin.disconnect() 

1805 

1806 return '' 

1807 

1808 @classmethod 

1809 def create_sr(cls, group_name, ips, redundancy, thin_provisioning, logger=default_logger.__func__): 

1810 """ 

1811 Create a new SR on the given nodes. 

1812 :param str group_name: The SR group_name to use. 

1813 :param set(str) ips: Node ips. 

1814 :param int redundancy: How many copy of volumes should we store? 

1815 :param bool thin_provisioning: Use thin or thick provisioning. 

1816 :param function logger: Function to log messages. 

1817 :return: A new LinstorSr instance. 

1818 :rtype: LinstorSr 

1819 """ 

1820 

1821 try: 

1822 cls._start_controller(start=True) 

1823 sr = cls._create_sr(group_name, ips, redundancy, thin_provisioning, logger) 

1824 finally: 

1825 # Controller must be stopped and volume unmounted because 

1826 # it is the role of the drbd-reactor daemon to do the right 

1827 # actions. 

1828 cls._start_controller(start=False) 

1829 cls._mount_volume( 

1830 cls.build_device_path(DATABASE_VOLUME_NAME), 

1831 DATABASE_PATH, 

1832 mount=False 

1833 ) 

1834 return sr 

1835 

1836 @classmethod 

1837 def _create_sr(cls, group_name, ips, redundancy, thin_provisioning, logger=default_logger.__func__): 

1838 # 1. Check if SR already exists. 

1839 uri = 'linstor://localhost' 

1840 

1841 lin = cls._create_linstor_instance(uri, keep_uri_unmodified=True) 

1842 

1843 node_names = list(ips.keys()) 

1844 for node_name, ip in ips.items(): 

1845 while True: 

1846 # Try to create node. 

1847 result = lin.node_create( 

1848 node_name, 

1849 linstor.consts.VAL_NODE_TYPE_CMBD, 

1850 ip 

1851 ) 

1852 

1853 errors = cls._filter_errors(result) 

1854 if cls._check_errors( 

1855 errors, [linstor.consts.FAIL_EXISTS_NODE] 

1856 ): 

1857 # If it already exists, remove, then recreate. 

1858 result = lin.node_delete(node_name) 

1859 error_str = cls._get_error_str(result) 

1860 if error_str: 

1861 raise LinstorVolumeManagerError( 

1862 'Failed to remove old node `{}`: {}' 

1863 .format(node_name, error_str) 

1864 ) 

1865 elif not errors: 

1866 break # Created! 

1867 else: 

1868 raise LinstorVolumeManagerError( 

1869 'Failed to create node `{}` with ip `{}`: {}'.format( 

1870 node_name, ip, cls._get_error_str(errors) 

1871 ) 

1872 ) 

1873 

1874 driver_pool_name = group_name 

1875 base_group_name = group_name 

1876 group_name = cls.build_group_name(group_name) 

1877 storage_pool_name = group_name 

1878 pools = lin.storage_pool_list_raise(filter_by_stor_pools=[storage_pool_name]).storage_pools 

1879 if pools: 

1880 existing_node_names = [pool.node_name for pool in pools] 

1881 raise LinstorVolumeManagerError( 

1882 'Unable to create SR `{}`. It already exists on node(s): {}' 

1883 .format(group_name, existing_node_names) 

1884 ) 

1885 

1886 if lin.resource_group_list_raise( 

1887 cls.get_all_group_names(base_group_name) 

1888 ).resource_groups: 

1889 if not lin.resource_dfn_list_raise().resource_definitions: 

1890 backup_path = cls._create_database_backup_path() 

1891 logger( 

1892 'Group name already exists `{}` without LVs. ' 

1893 'Ignoring and moving the config files in {}'.format(group_name, backup_path) 

1894 ) 

1895 cls._move_files(DATABASE_PATH, backup_path) 

1896 else: 

1897 raise LinstorVolumeManagerError( 

1898 'Unable to create SR `{}`: The group name already exists' 

1899 .format(group_name) 

1900 ) 

1901 

1902 if thin_provisioning: 

1903 driver_pool_parts = driver_pool_name.split('/') 

1904 if not len(driver_pool_parts) == 2: 

1905 raise LinstorVolumeManagerError( 

1906 'Invalid group name using thin provisioning. ' 

1907 'Expected format: \'VG/LV`\'' 

1908 ) 

1909 

1910 # 2. Create storage pool on each node + resource group. 

1911 reg_volume_group_not_found = re.compile( 

1912 ".*Volume group '.*' not found$" 

1913 ) 

1914 

1915 i = 0 

1916 try: 

1917 # 2.a. Create storage pools. 

1918 storage_pool_count = 0 

1919 while i < len(node_names): 

1920 node_name = node_names[i] 

1921 

1922 result = lin.storage_pool_create( 

1923 node_name=node_name, 

1924 storage_pool_name=storage_pool_name, 

1925 storage_driver='LVM_THIN' if thin_provisioning else 'LVM', 

1926 driver_pool_name=driver_pool_name 

1927 ) 

1928 

1929 errors = linstor.Linstor.filter_api_call_response_errors( 

1930 result 

1931 ) 

1932 if errors: 

1933 if len(errors) == 1 and errors[0].is_error( 

1934 linstor.consts.FAIL_STOR_POOL_CONFIGURATION_ERROR 

1935 ) and reg_volume_group_not_found.match(errors[0].message): 

1936 logger( 

1937 'Volume group `{}` not found on `{}`. Ignoring...' 

1938 .format(group_name, node_name) 

1939 ) 

1940 cls._destroy_storage_pool(lin, storage_pool_name, node_name) 

1941 else: 

1942 error_str = cls._get_error_str(result) 

1943 raise LinstorVolumeManagerError( 

1944 'Could not create SP `{}` on node `{}`: {}' 

1945 .format(group_name, node_name, error_str) 

1946 ) 

1947 else: 

1948 storage_pool_count += 1 

1949 i += 1 

1950 

1951 if not storage_pool_count: 

1952 raise LinstorVolumeManagerError( 

1953 'Unable to create SR `{}`: No VG group found'.format( 

1954 group_name, 

1955 ) 

1956 ) 

1957 

1958 # 2.b. Create resource groups. 

1959 ha_group_name = cls._build_ha_group_name(base_group_name) 

1960 cls._create_resource_group( 

1961 lin, 

1962 group_name, 

1963 storage_pool_name, 

1964 redundancy, 

1965 True 

1966 ) 

1967 cls._create_resource_group( 

1968 lin, 

1969 ha_group_name, 

1970 storage_pool_name, 

1971 3, 

1972 True 

1973 ) 

1974 

1975 # 3. Create the LINSTOR database volume and mount it. 

1976 try: 

1977 logger('Creating database volume...') 

1978 volume_path = cls._create_database_volume( 

1979 lin, ha_group_name, storage_pool_name, node_names, redundancy 

1980 ) 

1981 except LinstorVolumeManagerError as e: 

1982 if e.code != LinstorVolumeManagerError.ERR_VOLUME_EXISTS: 

1983 logger('Destroying database volume after creation fail...') 

1984 cls._force_destroy_database_volume(lin, group_name) 

1985 raise 

1986 

1987 try: 

1988 logger('Mounting database volume...') 

1989 

1990 # First we must disable the controller to move safely the 

1991 # LINSTOR config. 

1992 cls._start_controller(start=False) 

1993 

1994 cls._mount_database_volume(volume_path) 

1995 except Exception as e: 

1996 # Ensure we are connected because controller has been 

1997 # restarted during mount call. 

1998 logger('Destroying database volume after mount fail...') 

1999 

2000 try: 

2001 cls._start_controller(start=True) 

2002 except Exception: 

2003 pass 

2004 

2005 lin = cls._create_linstor_instance( 

2006 uri, keep_uri_unmodified=True 

2007 ) 

2008 cls._force_destroy_database_volume(lin, group_name) 

2009 raise e 

2010 

2011 cls._start_controller(start=True) 

2012 lin = cls._create_linstor_instance(uri, keep_uri_unmodified=True) 

2013 

2014 # 4. Remove storage pools/resource/volume group in the case of errors. 

2015 except Exception as e: 

2016 logger('Destroying resource group and storage pools after fail...') 

2017 try: 

2018 cls._destroy_resource_group(lin, group_name) 

2019 cls._destroy_resource_group(lin, ha_group_name) 

2020 except Exception as e2: 

2021 logger('Failed to destroy resource group: {}'.format(e2)) 

2022 pass 

2023 j = 0 

2024 i = min(i, len(node_names) - 1) 

2025 while j <= i: 

2026 try: 

2027 cls._destroy_storage_pool(lin, storage_pool_name, node_names[j]) 

2028 except Exception as e2: 

2029 logger('Failed to destroy resource group: {}'.format(e2)) 

2030 pass 

2031 j += 1 

2032 raise e 

2033 

2034 # 5. Return new instance. 

2035 instance = cls.__new__(cls) 

2036 instance._linstor = lin 

2037 instance._logger = logger 

2038 instance._redundancy = redundancy 

2039 instance._base_group_name = base_group_name 

2040 instance._group_name = group_name 

2041 instance._volumes = set() 

2042 instance._storage_pools_time = 0 

2043 instance._kv_cache = instance._create_kv_cache() 

2044 instance._resource_cache = None 

2045 instance._resource_cache_dirty = True 

2046 instance._volume_info_cache = None 

2047 instance._volume_info_cache_dirty = True 

2048 return instance 

2049 

2050 @classmethod 

2051 def build_device_path(cls, volume_name): 

2052 """ 

2053 Build a device path given a volume name. 

2054 :param str volume_name: The volume name to use. 

2055 :return: A valid or not device path. 

2056 :rtype: str 

2057 """ 

2058 

2059 return '{}{}/0'.format(cls.DEV_ROOT_PATH, volume_name) 

2060 

2061 @classmethod 

2062 def build_volume_name(cls, base_name): 

2063 """ 

2064 Build a volume name given a base name (i.e. a UUID). 

2065 :param str base_name: The volume name to use. 

2066 :return: A valid or not device path. 

2067 :rtype: str 

2068 """ 

2069 return '{}{}'.format(cls.PREFIX_VOLUME, base_name) 

2070 

2071 @classmethod 

2072 def round_up_volume_size(cls, volume_size): 

2073 """ 

2074 Align volume size on higher multiple of BLOCK_SIZE. 

2075 :param int volume_size: The volume size to align. 

2076 :return: An aligned volume size. 

2077 :rtype: int 

2078 """ 

2079 return round_up(volume_size, cls.BLOCK_SIZE) 

2080 

2081 @classmethod 

2082 def round_down_volume_size(cls, volume_size): 

2083 """ 

2084 Align volume size on lower multiple of BLOCK_SIZE. 

2085 :param int volume_size: The volume size to align. 

2086 :return: An aligned volume size. 

2087 :rtype: int 

2088 """ 

2089 return round_down(volume_size, cls.BLOCK_SIZE) 

2090 

2091 # -------------------------------------------------------------------------- 

2092 # Private helpers. 

2093 # -------------------------------------------------------------------------- 

2094 

2095 def _create_kv_cache(self): 

2096 self._kv_cache = self._create_linstor_kv('/') 

2097 self._kv_cache_dirty = False 

2098 return self._kv_cache 

2099 

2100 def _get_kv_cache(self): 

2101 if self._kv_cache_dirty: 

2102 self._kv_cache = self._create_kv_cache() 

2103 return self._kv_cache 

2104 

2105 def _create_resource_cache(self): 

2106 self._resource_cache = self._linstor.resource_list_raise() 

2107 self._resource_cache_dirty = False 

2108 return self._resource_cache 

2109 

2110 def _get_resource_cache(self): 

2111 if self._resource_cache_dirty: 

2112 self._resource_cache = self._create_resource_cache() 

2113 return self._resource_cache 

2114 

2115 def _mark_resource_cache_as_dirty(self): 

2116 self._resource_cache_dirty = True 

2117 self._volume_info_cache_dirty = True 

2118 

2119 # -------------------------------------------------------------------------- 

2120 

2121 def _ensure_volume_exists(self, volume_uuid): 

2122 if volume_uuid not in self._volumes: 

2123 raise LinstorVolumeManagerError( 

2124 'volume `{}` doesn\'t exist'.format(volume_uuid), 

2125 LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS 

2126 ) 

2127 

2128 def _find_best_size_candidates(self): 

2129 result = self._linstor.resource_group_qmvs(self._group_name) 

2130 error_str = self._get_error_str(result) 

2131 if error_str: 

2132 raise LinstorVolumeManagerError( 

2133 'Failed to get max volume size allowed of SR `{}`: {}'.format( 

2134 self._group_name, 

2135 error_str 

2136 ) 

2137 ) 

2138 return result[0].candidates 

2139 

2140 def _fetch_resource_names(self, ignore_deleted=True): 

2141 resource_names = set() 

2142 dfns = self._linstor.resource_dfn_list_raise().resource_definitions 

2143 for dfn in dfns: 

2144 if dfn.resource_group_name in self.get_all_group_names(self._base_group_name) and ( 

2145 ignore_deleted or 

2146 linstor.consts.FLAG_DELETE not in dfn.flags 

2147 ): 

2148 resource_names.add(dfn.name) 

2149 return resource_names 

2150 

2151 def _get_volumes_info(self, volume_names=None): 

2152 all_volume_info = {} 

2153 

2154 if not self._volume_info_cache_dirty: 

2155 return self._volume_info_cache 

2156 

2157 # `volume_names` MUST contain all volumes registered in the KV store. 

2158 # It can be provided to the function to avoid double fetching. 

2159 if not volume_names: 

2160 volume_names = self.get_volumes_with_name() 

2161 volume_names = set(volume_names.values()) 

2162 

2163 def process_resource(resource): 

2164 if resource.name not in all_volume_info: 

2165 current = all_volume_info[resource.name] = self.VolumeInfo( 

2166 resource.name 

2167 ) 

2168 else: 

2169 current = all_volume_info[resource.name] 

2170 

2171 if linstor.consts.FLAG_DISKLESS not in resource.flags: 

2172 current.diskful.append(resource.node_name) 

2173 

2174 for volume in resource.volumes: 

2175 # We ignore diskless pools of the form "DfltDisklessStorPool". 

2176 if volume.storage_pool_name != self._group_name: 

2177 continue 

2178 # Only fetch first volume. 

2179 if volume.number != 0: 

2180 continue 

2181 

2182 allocated_size = volume.allocated_size 

2183 if allocated_size > current.allocated_size: 

2184 current.allocated_size = allocated_size 

2185 

2186 usable_size = volume.usable_size 

2187 if usable_size > 0 and ( 

2188 usable_size < current.virtual_size or 

2189 not current.virtual_size 

2190 ): 

2191 current.virtual_size = usable_size 

2192 

2193 try: 

2194 for resource in self._get_resource_cache().resources: 

2195 if resource.name in volume_names: 

2196 process_resource(resource) 

2197 for volume in all_volume_info.values(): 

2198 if volume.allocated_size <= 0: 

2199 raise LinstorVolumeManagerError('Failed to get allocated size of `{}`'.format(resource.name)) 

2200 

2201 if volume.virtual_size <= 0: 

2202 raise LinstorVolumeManagerError('Failed to get usable size of `{}`'.format(volume.name)) 

2203 

2204 volume.allocated_size *= 1024 

2205 volume.virtual_size *= 1024 

2206 except LinstorVolumeManagerError: 

2207 self._mark_resource_cache_as_dirty() 

2208 raise 

2209 

2210 self._volume_info_cache_dirty = False 

2211 self._volume_info_cache = all_volume_info 

2212 

2213 return all_volume_info 

2214 

2215 def _get_volume_node_names_and_size(self, volume_name): 

2216 node_names = set() 

2217 size = -1 

2218 for resource in self._linstor.resource_list_raise( 

2219 filter_by_resources=[volume_name] 

2220 ).resources: 

2221 for volume in resource.volumes: 

2222 # We ignore diskless pools of the form "DfltDisklessStorPool". 

2223 if volume.storage_pool_name != self._group_name: 

2224 continue 

2225 

2226 node_names.add(resource.node_name) 

2227 

2228 usable_size = volume.usable_size 

2229 if usable_size <= 0: 

2230 continue 

2231 

2232 if size < 0: 

2233 size = usable_size 

2234 else: 

2235 size = min(size, usable_size) 

2236 

2237 if size <= 0: 

2238 raise LinstorVolumeManagerError('Failed to get usable size of `{}`'.format(resource.name)) 

2239 

2240 return (node_names, size * 1024) 

2241 

2242 def _compute_size(self, attr): 

2243 capacity = 0 

2244 for pool in self._get_storage_pools(force=True): 

2245 space = pool.free_space 

2246 if space: 

2247 size = getattr(space, attr) 

2248 if size < 0: 

2249 raise LinstorVolumeManagerError( 

2250 'Failed to get pool {} attr of `{}`' 

2251 .format(attr, pool.node_name) 

2252 ) 

2253 capacity += size 

2254 return capacity * 1024 

2255 

2256 def _get_node_names(self): 

2257 node_names = set() 

2258 for pool in self._get_storage_pools(): 

2259 node_names.add(pool.node_name) 

2260 return node_names 

2261 

2262 def _get_storage_pools(self, force=False): 

2263 cur_time = time.time() 

2264 elsaped_time = cur_time - self._storage_pools_time 

2265 

2266 if force or elsaped_time >= self.STORAGE_POOLS_FETCH_INTERVAL: 

2267 self._storage_pools = self._linstor.storage_pool_list_raise( 

2268 filter_by_stor_pools=[self._group_name] 

2269 ).storage_pools 

2270 self._storage_pools_time = time.time() 

2271 

2272 return self._storage_pools 

2273 

2274 def _create_volume( 

2275 self, 

2276 volume_uuid, 

2277 volume_name, 

2278 size, 

2279 place_resources, 

2280 high_availability 

2281 ): 

2282 size = self.round_up_volume_size(size) 

2283 self._mark_resource_cache_as_dirty() 

2284 

2285 group_name = self._ha_group_name if high_availability else self._group_name 

2286 def create_definition(): 

2287 first_attempt = True 

2288 while True: 

2289 try: 

2290 self._check_volume_creation_errors( 

2291 self._linstor.resource_group_spawn( 

2292 rsc_grp_name=group_name, 

2293 rsc_dfn_name=volume_name, 

2294 vlm_sizes=['{}B'.format(size)], 

2295 definitions_only=True 

2296 ), 

2297 volume_uuid, 

2298 self._group_name 

2299 ) 

2300 break 

2301 except LinstorVolumeManagerError as e: 

2302 if ( 

2303 not first_attempt or 

2304 not high_availability or 

2305 e.code != LinstorVolumeManagerError.ERR_GROUP_NOT_EXISTS 

2306 ): 

2307 raise 

2308 

2309 first_attempt = False 

2310 self._create_resource_group( 

2311 self._linstor, 

2312 group_name, 

2313 self._group_name, 

2314 3, 

2315 True 

2316 ) 

2317 

2318 self._configure_volume_peer_slots(self._linstor, volume_name) 

2319 

2320 def clean(): 

2321 try: 

2322 self._destroy_volume(volume_uuid, force=True, preserve_properties=True) 

2323 except Exception as e: 

2324 self._logger( 

2325 'Unable to destroy volume {} after creation fail: {}' 

2326 .format(volume_uuid, e) 

2327 ) 

2328 

2329 def create(): 

2330 try: 

2331 create_definition() 

2332 if place_resources: 

2333 # Basic case when we use the default redundancy of the group. 

2334 self._check_volume_creation_errors( 

2335 self._linstor.resource_auto_place( 

2336 rsc_name=volume_name, 

2337 place_count=self._redundancy, 

2338 diskless_on_remaining=False 

2339 ), 

2340 volume_uuid, 

2341 self._group_name 

2342 ) 

2343 except LinstorVolumeManagerError as e: 

2344 if e.code != LinstorVolumeManagerError.ERR_VOLUME_EXISTS: 

2345 clean() 

2346 raise 

2347 except Exception: 

2348 clean() 

2349 raise 

2350 

2351 util.retry(create, maxretry=5) 

2352 

2353 def _create_volume_with_properties( 

2354 self, 

2355 volume_uuid, 

2356 volume_name, 

2357 size, 

2358 place_resources, 

2359 high_availability 

2360 ): 

2361 if self.check_volume_exists(volume_uuid): 

2362 raise LinstorVolumeManagerError( 

2363 'Could not create volume `{}` from SR `{}`, it already exists' 

2364 .format(volume_uuid, self._group_name) + ' in properties', 

2365 LinstorVolumeManagerError.ERR_VOLUME_EXISTS 

2366 ) 

2367 

2368 if volume_name in self._fetch_resource_names(): 

2369 raise LinstorVolumeManagerError( 

2370 'Could not create volume `{}` from SR `{}`, '.format( 

2371 volume_uuid, self._group_name 

2372 ) + 'resource of the same name already exists in LINSTOR' 

2373 ) 

2374 

2375 # I am paranoid. 

2376 volume_properties = self._get_volume_properties(volume_uuid) 

2377 if (volume_properties.get(self.PROP_NOT_EXISTS) is not None): 

2378 raise LinstorVolumeManagerError( 

2379 'Could not create volume `{}`, '.format(volume_uuid) + 

2380 'properties already exist' 

2381 ) 

2382 

2383 try: 

2384 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_CREATING 

2385 volume_properties[self.PROP_VOLUME_NAME] = volume_name 

2386 

2387 self._create_volume( 

2388 volume_uuid, 

2389 volume_name, 

2390 size, 

2391 place_resources, 

2392 high_availability 

2393 ) 

2394 

2395 assert volume_properties.namespace == \ 

2396 self._build_volume_namespace(volume_uuid) 

2397 return volume_properties 

2398 except LinstorVolumeManagerError as e: 

2399 # Do not destroy existing resource! 

2400 # In theory we can't get this error because we check this event 

2401 # before the `self._create_volume` case. 

2402 # It can only happen if the same volume uuid is used in the same 

2403 # call in another host. 

2404 if e.code != LinstorVolumeManagerError.ERR_VOLUME_EXISTS: 

2405 self._destroy_volume(volume_uuid, force=True) 

2406 raise 

2407 

2408 def _find_device_path(self, volume_uuid, volume_name): 

2409 current_device_path = self._request_device_path( 

2410 volume_uuid, volume_name, activate=True 

2411 ) 

2412 

2413 # We use realpath here to get the /dev/drbd<id> path instead of 

2414 # /dev/drbd/by-res/<resource_name>. 

2415 expected_device_path = self.build_device_path(volume_name) 

2416 util.wait_for_path(expected_device_path, 5) 

2417 

2418 device_realpath = os.path.realpath(expected_device_path) 

2419 if current_device_path != device_realpath: 

2420 raise LinstorVolumeManagerError( 

2421 'Invalid path, current={}, expected={} (realpath={})' 

2422 .format( 

2423 current_device_path, 

2424 expected_device_path, 

2425 device_realpath 

2426 ) 

2427 ) 

2428 return expected_device_path 

2429 

2430 def _request_device_path(self, volume_uuid, volume_name, activate=False): 

2431 node_name = socket.gethostname() 

2432 

2433 resource = next(filter( 

2434 lambda resource: resource.node_name == node_name and 

2435 resource.name == volume_name, 

2436 self._get_resource_cache().resources 

2437 ), None) 

2438 

2439 if not resource: 

2440 if activate: 

2441 self._mark_resource_cache_as_dirty() 

2442 self._activate_device_path( 

2443 self._linstor, node_name, volume_name 

2444 ) 

2445 return self._request_device_path(volume_uuid, volume_name) 

2446 raise LinstorVolumeManagerError( 

2447 'Unable to get dev path for `{}`, no resource found but definition "seems" to exist' 

2448 .format(volume_uuid) 

2449 ) 

2450 

2451 # Contains a path of the /dev/drbd<id> form. 

2452 device_path = resource.volumes[0].device_path 

2453 if not device_path: 

2454 raise LinstorVolumeManagerError('Empty dev path for `{}`!'.format(volume_uuid)) 

2455 return device_path 

2456 

2457 def _destroy_resource(self, resource_name, force=False): 

2458 result = self._linstor.resource_dfn_delete(resource_name) 

2459 error_str = self._get_error_str(result) 

2460 if not error_str: 

2461 self._mark_resource_cache_as_dirty() 

2462 return 

2463 

2464 if not force: 

2465 self._mark_resource_cache_as_dirty() 

2466 raise LinstorVolumeManagerError( 

2467 'Could not destroy resource `{}` from SR `{}`: {}' 

2468 .format(resource_name, self._group_name, error_str) 

2469 ) 

2470 

2471 # If force is used, ensure there is no opener. 

2472 openers = get_all_volume_openers(resource_name, '0') 

2473 for host_openers in openers.values(): 

2474 if host_openers: 

2475 self._mark_resource_cache_as_dirty() 

2476 raise LinstorVolumeManagerError( 

2477 'Could not force destroy resource `{}` from SR `{}`: {} (openers=`{}`)' 

2478 .format(resource_name, self._group_name, error_str, openers) 

2479 ) 

2480 

2481 # Maybe the resource is blocked in primary mode. DRBD/LINSTOR issue? 

2482 resource_states = filter( 

2483 lambda resource_state: resource_state.name == resource_name, 

2484 self._get_resource_cache().resource_states 

2485 ) 

2486 

2487 # Mark only after computation of states. 

2488 self._mark_resource_cache_as_dirty() 

2489 

2490 for resource_state in resource_states: 

2491 volume_state = resource_state.volume_states[0] 

2492 if resource_state.in_use: 

2493 demote_drbd_resource(resource_state.node_name, resource_name) 

2494 break 

2495 self._destroy_resource(resource_name) 

2496 

2497 def _destroy_volume(self, volume_uuid, force=False, preserve_properties=False): 

2498 volume_properties = self._get_volume_properties(volume_uuid) 

2499 try: 

2500 volume_name = volume_properties.get(self.PROP_VOLUME_NAME) 

2501 if volume_name in self._fetch_resource_names(): 

2502 self._destroy_resource(volume_name, force) 

2503 

2504 # Assume this call is atomic. 

2505 if not preserve_properties: 

2506 volume_properties.clear() 

2507 except Exception as e: 

2508 raise LinstorVolumeManagerError( 

2509 'Cannot destroy volume `{}`: {}'.format(volume_uuid, e) 

2510 ) 

2511 

2512 def _build_volumes(self, repair): 

2513 properties = self._kv_cache 

2514 resource_names = self._fetch_resource_names() 

2515 

2516 self._volumes = set() 

2517 

2518 updating_uuid_volumes = self._get_volumes_by_property( 

2519 self.REG_UPDATING_UUID_SRC, ignore_inexisting_volumes=False 

2520 ) 

2521 if updating_uuid_volumes and not repair: 

2522 raise LinstorVolumeManagerError( 

2523 'Cannot build LINSTOR volume list: ' 

2524 'It exists invalid "updating uuid volumes", repair is required' 

2525 ) 

2526 

2527 existing_volumes = self._get_volumes_by_property( 

2528 self.REG_NOT_EXISTS, ignore_inexisting_volumes=False 

2529 ) 

2530 for volume_uuid, not_exists in existing_volumes.items(): 

2531 properties.namespace = self._build_volume_namespace(volume_uuid) 

2532 

2533 src_uuid = properties.get(self.PROP_UPDATING_UUID_SRC) 

2534 if src_uuid: 

2535 self._logger( 

2536 'Ignoring volume during manager initialization with prop ' 

2537 ' PROP_UPDATING_UUID_SRC: {} (properties={})' 

2538 .format( 

2539 volume_uuid, 

2540 self._get_filtered_properties(properties) 

2541 ) 

2542 ) 

2543 continue 

2544 

2545 # Insert volume in list if the volume exists. Or if the volume 

2546 # is being created and a slave wants to use it (repair = False). 

2547 # 

2548 # If we are on the master and if repair is True and state is 

2549 # Creating, it's probably a bug or crash: the creation process has 

2550 # been stopped. 

2551 if not_exists == self.STATE_EXISTS or ( 

2552 not repair and not_exists == self.STATE_CREATING 

2553 ): 

2554 self._volumes.add(volume_uuid) 

2555 continue 

2556 

2557 if not repair: 

2558 self._logger( 

2559 'Ignoring bad volume during manager initialization: {} ' 

2560 '(properties={})'.format( 

2561 volume_uuid, 

2562 self._get_filtered_properties(properties) 

2563 ) 

2564 ) 

2565 continue 

2566 

2567 # Remove bad volume. 

2568 try: 

2569 self._logger( 

2570 'Removing bad volume during manager initialization: {} ' 

2571 '(properties={})'.format( 

2572 volume_uuid, 

2573 self._get_filtered_properties(properties) 

2574 ) 

2575 ) 

2576 volume_name = properties.get(self.PROP_VOLUME_NAME) 

2577 

2578 # Little optimization, don't call `self._destroy_volume`, 

2579 # we already have resource name list. 

2580 if volume_name in resource_names: 

2581 self._destroy_resource(volume_name, force=True) 

2582 

2583 # Assume this call is atomic. 

2584 properties.clear() 

2585 except Exception as e: 

2586 # Do not raise, we don't want to block user action. 

2587 self._logger( 

2588 'Cannot clean volume {}: {}'.format(volume_uuid, e) 

2589 ) 

2590 

2591 # The volume can't be removed, maybe it's still in use, 

2592 # in this case rename it with the "DELETED_" prefix. 

2593 # This prefix is mandatory if it exists a snap transaction to 

2594 # rollback because the original VDI UUID can try to be renamed 

2595 # with the UUID we are trying to delete... 

2596 if not volume_uuid.startswith('DELETED_'): 

2597 self.update_volume_uuid( 

2598 volume_uuid, 'DELETED_' + volume_uuid, force=True 

2599 ) 

2600 

2601 for dest_uuid, src_uuid in updating_uuid_volumes.items(): 

2602 dest_namespace = self._build_volume_namespace(dest_uuid) 

2603 

2604 properties.namespace = dest_namespace 

2605 if int(properties.get(self.PROP_NOT_EXISTS)): 

2606 properties.clear() 

2607 continue 

2608 

2609 properties.namespace = self._build_volume_namespace(src_uuid) 

2610 properties.clear() 

2611 

2612 properties.namespace = dest_namespace 

2613 properties.pop(self.PROP_UPDATING_UUID_SRC) 

2614 

2615 if src_uuid in self._volumes: 

2616 self._volumes.remove(src_uuid) 

2617 self._volumes.add(dest_uuid) 

2618 

2619 def _get_sr_properties(self): 

2620 return self._create_linstor_kv(self._build_sr_namespace()) 

2621 

2622 def _get_volumes_by_property( 

2623 self, reg_prop, ignore_inexisting_volumes=True 

2624 ): 

2625 base_properties = self._get_kv_cache() 

2626 base_properties.namespace = self._build_volume_namespace() 

2627 

2628 volume_properties = {} 

2629 for volume_uuid in self._volumes: 

2630 volume_properties[volume_uuid] = '' 

2631 

2632 for key, value in base_properties.items(): 

2633 res = reg_prop.match(key) 

2634 if res: 

2635 volume_uuid = res.groups()[0] 

2636 if not ignore_inexisting_volumes or \ 

2637 volume_uuid in self._volumes: 

2638 volume_properties[volume_uuid] = value 

2639 

2640 return volume_properties 

2641 

2642 def _create_linstor_kv(self, namespace): 

2643 return linstor.KV( 

2644 self._group_name, 

2645 uri=self._linstor.controller_host(), 

2646 namespace=namespace 

2647 ) 

2648 

2649 def _get_volume_properties(self, volume_uuid): 

2650 properties = self._get_kv_cache() 

2651 properties.namespace = self._build_volume_namespace(volume_uuid) 

2652 return properties 

2653 

2654 @classmethod 

2655 def _build_sr_namespace(cls): 

2656 return '/{}/'.format(cls.NAMESPACE_SR) 

2657 

2658 @classmethod 

2659 def _build_volume_namespace(cls, volume_uuid=None): 

2660 # Return a path to all volumes if `volume_uuid` is not given. 

2661 if volume_uuid is None: 

2662 return '/{}/'.format(cls.NAMESPACE_VOLUME) 

2663 return '/{}/{}/'.format(cls.NAMESPACE_VOLUME, volume_uuid) 

2664 

2665 @classmethod 

2666 def _get_error_str(cls, result): 

2667 return ', '.join([ 

2668 err.message for err in cls._filter_errors(result) 

2669 ]) 

2670 

2671 @classmethod 

2672 def _create_linstor_instance( 

2673 cls, uri, keep_uri_unmodified=False, attempt_count=30 

2674 ): 

2675 retry = False 

2676 

2677 def connect(uri): 

2678 if not uri: 

2679 uri = get_controller_uri() 

2680 if not uri: 

2681 raise LinstorVolumeManagerError( 

2682 'Unable to find controller uri...' 

2683 ) 

2684 instance = linstor.Linstor(uri, keep_alive=True) 

2685 instance.connect() 

2686 return instance 

2687 

2688 try: 

2689 return connect(uri) 

2690 except (linstor.errors.LinstorNetworkError, LinstorVolumeManagerError): 

2691 pass 

2692 

2693 if not keep_uri_unmodified: 

2694 uri = None 

2695 

2696 return util.retry( 

2697 lambda: connect(uri), 

2698 maxretry=attempt_count, 

2699 period=1, 

2700 exceptions=[ 

2701 linstor.errors.LinstorNetworkError, 

2702 LinstorVolumeManagerError 

2703 ] 

2704 ) 

2705 

2706 @classmethod 

2707 def _configure_volume_peer_slots(cls, lin, volume_name): 

2708 result = lin.resource_dfn_modify(volume_name, {}, peer_slots=3) 

2709 error_str = cls._get_error_str(result) 

2710 if error_str: 

2711 raise LinstorVolumeManagerError( 

2712 'Could not configure volume peer slots of {}: {}' 

2713 .format(volume_name, error_str) 

2714 ) 

2715 

2716 @classmethod 

2717 def _activate_device_path(cls, lin, node_name, volume_name): 

2718 result = lin.resource_make_available(node_name, volume_name, diskful=False) 

2719 if linstor.Linstor.all_api_responses_no_error(result): 

2720 return 

2721 errors = linstor.Linstor.filter_api_call_response_errors(result) 

2722 if len(errors) == 1 and errors[0].is_error( 

2723 linstor.consts.FAIL_EXISTS_RSC 

2724 ): 

2725 return 

2726 

2727 raise LinstorVolumeManagerError( 

2728 'Unable to activate device path of `{}` on node `{}`: {}' 

2729 .format(volume_name, node_name, ', '.join( 

2730 [str(x) for x in result])) 

2731 ) 

2732 

2733 @classmethod 

2734 def _request_database_path(cls, lin, activate=False): 

2735 node_name = socket.gethostname() 

2736 

2737 try: 

2738 resource = next(filter( 

2739 lambda resource: resource.node_name == node_name and 

2740 resource.name == DATABASE_VOLUME_NAME, 

2741 lin.resource_list_raise().resources 

2742 ), None) 

2743 except Exception as e: 

2744 raise LinstorVolumeManagerError( 

2745 'Unable to fetch database resource: {}' 

2746 .format(e) 

2747 ) 

2748 

2749 if not resource: 

2750 if activate: 

2751 cls._activate_device_path( 

2752 lin, node_name, DATABASE_VOLUME_NAME 

2753 ) 

2754 return cls._request_database_path( 

2755 DATABASE_VOLUME_NAME, DATABASE_VOLUME_NAME 

2756 ) 

2757 raise LinstorVolumeManagerError( 

2758 'Empty dev path for `{}`, but definition "seems" to exist' 

2759 .format(DATABASE_PATH) 

2760 ) 

2761 # Contains a path of the /dev/drbd<id> form. 

2762 return resource.volumes[0].device_path 

2763 

2764 @classmethod 

2765 def _create_database_volume( 

2766 cls, lin, group_name, storage_pool_name, node_names, redundancy 

2767 ): 

2768 try: 

2769 dfns = lin.resource_dfn_list_raise().resource_definitions 

2770 except Exception as e: 

2771 raise LinstorVolumeManagerError( 

2772 'Unable to get definitions during database creation: {}' 

2773 .format(e) 

2774 ) 

2775 

2776 if dfns: 

2777 raise LinstorVolumeManagerError( 

2778 'Could not create volume `{}` from SR `{}`, '.format( 

2779 DATABASE_VOLUME_NAME, group_name 

2780 ) + 'LINSTOR volume list must be empty.' 

2781 ) 

2782 

2783 # Workaround to use thin lvm. Without this line an error is returned: 

2784 # "Not enough available nodes" 

2785 # I don't understand why but this command protect against this bug. 

2786 try: 

2787 pools = lin.storage_pool_list_raise( 

2788 filter_by_stor_pools=[storage_pool_name] 

2789 ) 

2790 except Exception as e: 

2791 raise LinstorVolumeManagerError( 

2792 'Failed to get storage pool list before database creation: {}' 

2793 .format(e) 

2794 ) 

2795 

2796 # Ensure we have a correct list of storage pools. 

2797 assert pools.storage_pools # We must have at least one storage pool! 

2798 nodes_with_pool = list(map(lambda pool: pool.node_name, pools.storage_pools)) 

2799 for node_name in nodes_with_pool: 

2800 assert node_name in node_names 

2801 util.SMlog('Nodes with storage pool: {}'.format(nodes_with_pool)) 

2802 

2803 # Create the database definition. 

2804 size = cls.round_up_volume_size(DATABASE_SIZE) 

2805 cls._check_volume_creation_errors(lin.resource_group_spawn( 

2806 rsc_grp_name=group_name, 

2807 rsc_dfn_name=DATABASE_VOLUME_NAME, 

2808 vlm_sizes=['{}B'.format(size)], 

2809 definitions_only=True 

2810 ), DATABASE_VOLUME_NAME, group_name) 

2811 cls._configure_volume_peer_slots(lin, DATABASE_VOLUME_NAME) 

2812 

2813 # Create real resources on the first nodes. 

2814 resources = [] 

2815 

2816 diskful_nodes = [] 

2817 diskless_nodes = [] 

2818 for node_name in node_names: 

2819 if node_name in nodes_with_pool: 

2820 diskful_nodes.append(node_name) 

2821 else: 

2822 diskless_nodes.append(node_name) 

2823 

2824 assert diskful_nodes 

2825 for node_name in diskful_nodes[:redundancy]: 

2826 util.SMlog('Create database diskful on {}'.format(node_name)) 

2827 resources.append(linstor.ResourceData( 

2828 node_name=node_name, 

2829 rsc_name=DATABASE_VOLUME_NAME, 

2830 storage_pool=storage_pool_name 

2831 )) 

2832 # Create diskless resources on the remaining set. 

2833 for node_name in diskful_nodes[redundancy:] + diskless_nodes: 

2834 util.SMlog('Create database diskless on {}'.format(node_name)) 

2835 resources.append(linstor.ResourceData( 

2836 node_name=node_name, 

2837 rsc_name=DATABASE_VOLUME_NAME, 

2838 diskless=True 

2839 )) 

2840 

2841 result = lin.resource_create(resources) 

2842 error_str = cls._get_error_str(result) 

2843 if error_str: 

2844 raise LinstorVolumeManagerError( 

2845 'Could not create database volume from SR `{}`: {}'.format( 

2846 group_name, error_str 

2847 ) 

2848 ) 

2849 

2850 # Create database and ensure path exists locally and 

2851 # on replicated devices. 

2852 current_device_path = cls._request_database_path(lin, activate=True) 

2853 

2854 # Ensure diskless paths exist on other hosts. Otherwise PBDs can't be 

2855 # plugged. 

2856 for node_name in node_names: 

2857 cls._activate_device_path(lin, node_name, DATABASE_VOLUME_NAME) 

2858 

2859 # We use realpath here to get the /dev/drbd<id> path instead of 

2860 # /dev/drbd/by-res/<resource_name>. 

2861 expected_device_path = cls.build_device_path(DATABASE_VOLUME_NAME) 

2862 util.wait_for_path(expected_device_path, 5) 

2863 

2864 device_realpath = os.path.realpath(expected_device_path) 

2865 if current_device_path != device_realpath: 

2866 raise LinstorVolumeManagerError( 

2867 'Invalid path, current={}, expected={} (realpath={})' 

2868 .format( 

2869 current_device_path, 

2870 expected_device_path, 

2871 device_realpath 

2872 ) 

2873 ) 

2874 

2875 try: 

2876 util.retry( 

2877 lambda: util.pread2([DATABASE_MKFS, expected_device_path]), 

2878 maxretry=5 

2879 ) 

2880 except Exception as e: 

2881 raise LinstorVolumeManagerError( 

2882 'Failed to execute {} on database volume: {}' 

2883 .format(DATABASE_MKFS, e) 

2884 ) 

2885 

2886 return expected_device_path 

2887 

2888 @classmethod 

2889 def _destroy_database_volume(cls, lin, group_name): 

2890 error_str = cls._get_error_str( 

2891 lin.resource_dfn_delete(DATABASE_VOLUME_NAME) 

2892 ) 

2893 if error_str: 

2894 raise LinstorVolumeManagerError( 

2895 'Could not destroy resource `{}` from SR `{}`: {}' 

2896 .format(DATABASE_VOLUME_NAME, group_name, error_str) 

2897 ) 

2898 

2899 @classmethod 

2900 def _mount_database_volume(cls, volume_path, mount=True, force=False): 

2901 try: 

2902 # 1. Create a backup config folder. 

2903 database_not_empty = bool(os.listdir(DATABASE_PATH)) 

2904 backup_path = cls._create_database_backup_path() 

2905 

2906 # 2. Move the config in the mounted volume. 

2907 if database_not_empty: 

2908 cls._move_files(DATABASE_PATH, backup_path) 

2909 

2910 cls._mount_volume(volume_path, DATABASE_PATH, mount) 

2911 

2912 if database_not_empty: 

2913 cls._move_files(backup_path, DATABASE_PATH, force) 

2914 

2915 # 3. Remove useless backup directory. 

2916 try: 

2917 os.rmdir(backup_path) 

2918 except Exception as e: 

2919 raise LinstorVolumeManagerError( 

2920 'Failed to remove backup path {} of LINSTOR config: {}' 

2921 .format(backup_path, e) 

2922 ) 

2923 except Exception as e: 

2924 def force_exec(fn): 

2925 try: 

2926 fn() 

2927 except Exception: 

2928 pass 

2929 

2930 if mount == cls._is_mounted(DATABASE_PATH): 

2931 force_exec(lambda: cls._move_files( 

2932 DATABASE_PATH, backup_path 

2933 )) 

2934 force_exec(lambda: cls._mount_volume( 

2935 volume_path, DATABASE_PATH, not mount 

2936 )) 

2937 

2938 if mount != cls._is_mounted(DATABASE_PATH): 

2939 force_exec(lambda: cls._move_files( 

2940 backup_path, DATABASE_PATH 

2941 )) 

2942 

2943 force_exec(lambda: os.rmdir(backup_path)) 

2944 raise e 

2945 

2946 @classmethod 

2947 def _force_destroy_database_volume(cls, lin, group_name): 

2948 try: 

2949 cls._destroy_database_volume(lin, group_name) 

2950 except Exception: 

2951 pass 

2952 

2953 @classmethod 

2954 def _destroy_storage_pool(cls, lin, group_name, node_name): 

2955 def destroy(): 

2956 result = lin.storage_pool_delete(node_name, group_name) 

2957 errors = cls._filter_errors(result) 

2958 if cls._check_errors(errors, [ 

2959 linstor.consts.FAIL_NOT_FOUND_STOR_POOL, 

2960 linstor.consts.FAIL_NOT_FOUND_STOR_POOL_DFN 

2961 ]): 

2962 return 

2963 

2964 if errors: 

2965 raise LinstorVolumeManagerError( 

2966 'Failed to destroy SP `{}` on node `{}`: {}'.format( 

2967 group_name, 

2968 node_name, 

2969 cls._get_error_str(errors) 

2970 ) 

2971 ) 

2972 

2973 # We must retry to avoid errors like: 

2974 # "can not be deleted as volumes / snapshot-volumes are still using it" 

2975 # after LINSTOR database volume destruction. 

2976 return util.retry(destroy, maxretry=10) 

2977 

2978 @classmethod 

2979 def _create_resource_group( 

2980 cls, 

2981 lin, 

2982 group_name, 

2983 storage_pool_name, 

2984 redundancy, 

2985 destroy_old_group 

2986 ): 

2987 rg_creation_attempt = 0 

2988 while True: 

2989 result = lin.resource_group_create( 

2990 name=group_name, 

2991 place_count=redundancy, 

2992 storage_pool=storage_pool_name, 

2993 diskless_on_remaining=False 

2994 ) 

2995 error_str = cls._get_error_str(result) 

2996 if not error_str: 

2997 break 

2998 

2999 errors = cls._filter_errors(result) 

3000 if destroy_old_group and cls._check_errors(errors, [ 

3001 linstor.consts.FAIL_EXISTS_RSC_GRP 

3002 ]): 

3003 rg_creation_attempt += 1 

3004 if rg_creation_attempt < 2: 

3005 try: 

3006 cls._destroy_resource_group(lin, group_name) 

3007 except Exception as e: 

3008 error_str = 'Failed to destroy old and empty RG: {}'.format(e) 

3009 else: 

3010 continue 

3011 

3012 raise LinstorVolumeManagerError( 

3013 'Could not create RG `{}`: {}'.format( 

3014 group_name, error_str 

3015 ) 

3016 ) 

3017 

3018 result = lin.volume_group_create(group_name) 

3019 error_str = cls._get_error_str(result) 

3020 if error_str: 

3021 raise LinstorVolumeManagerError( 

3022 'Could not create VG `{}`: {}'.format( 

3023 group_name, error_str 

3024 ) 

3025 ) 

3026 

3027 @classmethod 

3028 def _destroy_resource_group(cls, lin, group_name): 

3029 def destroy(): 

3030 result = lin.resource_group_delete(group_name) 

3031 errors = cls._filter_errors(result) 

3032 if cls._check_errors(errors, [ 

3033 linstor.consts.FAIL_NOT_FOUND_RSC_GRP 

3034 ]): 

3035 return 

3036 

3037 if errors: 

3038 raise LinstorVolumeManagerError( 

3039 'Failed to destroy RG `{}`: {}' 

3040 .format(group_name, cls._get_error_str(errors)) 

3041 ) 

3042 

3043 return util.retry(destroy, maxretry=10) 

3044 

3045 @classmethod 

3046 def build_group_name(cls, base_name): 

3047 # If thin provisioning is used we have a path like this: 

3048 # `VG/LV`. "/" is not accepted by LINSTOR. 

3049 return '{}{}'.format(cls.PREFIX_SR, base_name.replace('/', '_')) 

3050 

3051 # Used to store important data in a HA context, 

3052 # i.e. a replication count of 3. 

3053 @classmethod 

3054 def _build_ha_group_name(cls, base_name): 

3055 return '{}{}'.format(cls.PREFIX_HA, base_name.replace('/', '_')) 

3056 

3057 @classmethod 

3058 def _check_volume_creation_errors(cls, result, volume_uuid, group_name): 

3059 errors = cls._filter_errors(result) 

3060 if cls._check_errors(errors, [ 

3061 linstor.consts.FAIL_EXISTS_RSC, linstor.consts.FAIL_EXISTS_RSC_DFN 

3062 ]): 

3063 raise LinstorVolumeManagerError( 

3064 'Failed to create volume `{}` from SR `{}`, it already exists' 

3065 .format(volume_uuid, group_name), 

3066 LinstorVolumeManagerError.ERR_VOLUME_EXISTS 

3067 ) 

3068 

3069 if cls._check_errors(errors, [linstor.consts.FAIL_NOT_FOUND_RSC_GRP]): 

3070 raise LinstorVolumeManagerError( 

3071 'Failed to create volume `{}` from SR `{}`, resource group doesn\'t exist' 

3072 .format(volume_uuid, group_name), 

3073 LinstorVolumeManagerError.ERR_GROUP_NOT_EXISTS 

3074 ) 

3075 

3076 if errors: 

3077 raise LinstorVolumeManagerError( 

3078 'Failed to create volume `{}` from SR `{}`: {}'.format( 

3079 volume_uuid, 

3080 group_name, 

3081 cls._get_error_str(errors) 

3082 ) 

3083 ) 

3084 

3085 @classmethod 

3086 def _move_files(cls, src_dir, dest_dir, force=False): 

3087 def listdir(dir): 

3088 ignored = ['lost+found'] 

3089 return [file for file in os.listdir(dir) if file not in ignored] 

3090 

3091 try: 

3092 if not force: 

3093 files = listdir(dest_dir) 

3094 if files: 

3095 raise LinstorVolumeManagerError( 

3096 'Cannot move files from {} to {} because destination ' 

3097 'contains: {}'.format(src_dir, dest_dir, files) 

3098 ) 

3099 except LinstorVolumeManagerError: 

3100 raise 

3101 except Exception as e: 

3102 raise LinstorVolumeManagerError( 

3103 'Cannot list dir {}: {}'.format(dest_dir, e) 

3104 ) 

3105 

3106 try: 

3107 for file in listdir(src_dir): 

3108 try: 

3109 dest_file = os.path.join(dest_dir, file) 

3110 if not force and os.path.exists(dest_file): 

3111 raise LinstorVolumeManagerError( 

3112 'Cannot move {} because it already exists in the ' 

3113 'destination'.format(file) 

3114 ) 

3115 shutil.move(os.path.join(src_dir, file), dest_file) 

3116 except LinstorVolumeManagerError: 

3117 raise 

3118 except Exception as e: 

3119 raise LinstorVolumeManagerError( 

3120 'Cannot move {}: {}'.format(file, e) 

3121 ) 

3122 except Exception as e: 

3123 if not force: 

3124 try: 

3125 cls._move_files(dest_dir, src_dir, force=True) 

3126 except Exception: 

3127 pass 

3128 

3129 raise LinstorVolumeManagerError( 

3130 'Failed to move files from {} to {}: {}'.format( 

3131 src_dir, dest_dir, e 

3132 ) 

3133 ) 

3134 

3135 @staticmethod 

3136 def _create_database_backup_path(): 

3137 path = DATABASE_PATH + '-' + str(uuid.uuid4()) 

3138 try: 

3139 os.mkdir(path) 

3140 return path 

3141 except Exception as e: 

3142 raise LinstorVolumeManagerError( 

3143 'Failed to create backup path {} of LINSTOR config: {}' 

3144 .format(path, e) 

3145 ) 

3146 

3147 @staticmethod 

3148 def _get_filtered_properties(properties): 

3149 return dict(properties.items()) 

3150 

3151 @staticmethod 

3152 def _filter_errors(result): 

3153 return [ 

3154 err for err in result 

3155 if hasattr(err, 'is_error') and err.is_error() 

3156 ] 

3157 

3158 @staticmethod 

3159 def _check_errors(result, codes): 

3160 for err in result: 

3161 for code in codes: 

3162 if err.is_error(code): 

3163 return True 

3164 return False 

3165 

3166 @classmethod 

3167 def _controller_is_running(cls): 

3168 return cls._service_is_running('linstor-controller') 

3169 

3170 @classmethod 

3171 def _start_controller(cls, start=True): 

3172 return cls._start_service('linstor-controller', start) 

3173 

3174 @staticmethod 

3175 def _start_service(name, start=True): 

3176 action = 'start' if start else 'stop' 

3177 (ret, out, err) = util.doexec([ 

3178 'systemctl', action, name 

3179 ]) 

3180 if ret != 0: 

3181 raise LinstorVolumeManagerError( 

3182 'Failed to {} {}: {} {}' 

3183 .format(action, name, out, err) 

3184 ) 

3185 

3186 @staticmethod 

3187 def _service_is_running(name): 

3188 (ret, out, err) = util.doexec([ 

3189 'systemctl', 'is-active', '--quiet', name 

3190 ]) 

3191 return not ret 

3192 

3193 @staticmethod 

3194 def _is_mounted(mountpoint): 

3195 (ret, out, err) = util.doexec(['mountpoint', '-q', mountpoint]) 

3196 return ret == 0 

3197 

3198 @classmethod 

3199 def _mount_volume(cls, volume_path, mountpoint, mount=True): 

3200 if mount: 

3201 try: 

3202 util.pread(['mount', volume_path, mountpoint]) 

3203 except Exception as e: 

3204 raise LinstorVolumeManagerError( 

3205 'Failed to mount volume {} on {}: {}' 

3206 .format(volume_path, mountpoint, e) 

3207 ) 

3208 else: 

3209 try: 

3210 if cls._is_mounted(mountpoint): 

3211 util.pread(['umount', mountpoint]) 

3212 except Exception as e: 

3213 raise LinstorVolumeManagerError( 

3214 'Failed to umount volume {} on {}: {}' 

3215 .format(volume_path, mountpoint, e) 

3216 ) 

3217 

3218 

3219# ============================================================================== 

3220 

3221# Check if a path is a DRBD resource and log the process name/pid 

3222# that opened it. 

3223def log_drbd_openers(path): 

3224 # Ignore if it's not a symlink to DRBD resource. 

3225 if not path.startswith(DRBD_BY_RES_PATH): 

3226 return 

3227 

3228 # Compute resource name. 

3229 res_name_end = path.find('/', len(DRBD_BY_RES_PATH)) 

3230 if res_name_end == -1: 

3231 return 

3232 res_name = path[len(DRBD_BY_RES_PATH):res_name_end] 

3233 

3234 volume_end = path.rfind('/') 

3235 if volume_end == res_name_end: 

3236 return 

3237 volume = path[volume_end + 1:] 

3238 

3239 try: 

3240 # Ensure path is a DRBD. 

3241 drbd_path = os.path.realpath(path) 

3242 stats = os.stat(drbd_path) 

3243 if not stat.S_ISBLK(stats.st_mode) or os.major(stats.st_rdev) != 147: 

3244 return 

3245 

3246 # Find where the device is open. 

3247 (ret, stdout, stderr) = util.doexec(['drbdadm', 'status', res_name]) 

3248 if ret != 0: 

3249 util.SMlog('Failed to execute `drbdadm status` on `{}`: {}'.format( 

3250 res_name, stderr 

3251 )) 

3252 return 

3253 

3254 # Is it a local device? 

3255 if stdout.startswith('{} role:Primary'.format(res_name)): 

3256 util.SMlog( 

3257 'DRBD resource `{}` is open on local host: {}' 

3258 .format(path, get_local_volume_openers(res_name, volume)) 

3259 ) 

3260 return 

3261 

3262 # Is it a remote device? 

3263 util.SMlog( 

3264 'DRBD resource `{}` is open on hosts: {}' 

3265 .format(path, get_all_volume_openers(res_name, volume)) 

3266 ) 

3267 except Exception as e: 

3268 util.SMlog( 

3269 'Got exception while trying to determine where DRBD resource ' + 

3270 '`{}` is open: {}'.format(path, e) 

3271 )