From e1e58aeb89e18f1cdfaaadc3be83fd600f10f33e Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 12:06:04 +0200 Subject: [PATCH 01/84] Remove references to obsolete flag ZFS_ENCRYPT_RPOOL --- README.md | 1 - install-zfs.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/README.md b/README.md index 77cf852..6654515 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,6 @@ The procedure can be entirely automated via environment variables: - ZFS_OS_INSTALLATION_SCRIPT : path of a script to execute instead of Ubiquity (see dedicated section below) - ZFS_SELECTED_DISKS : full path of the devices to create the pool on, comma-separated -- ZFS_ENCRYPT_RPOOL : set 1 to encrypt the pool - ZFS_PASSPHRASE - ZFS_BPOOL_NAME - ZFS_RPOOL_NAME diff --git a/install-zfs.sh b/install-zfs.sh index 48bb080..20e8367 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -171,7 +171,6 @@ The procedure can be entirely automated via environment variables: - ZFS_OS_INSTALLATION_SCRIPT : path of a script to execute instead of Ubiquity (see dedicated section below) - ZFS_SELECTED_DISKS : full path of the devices to create the pool on, comma-separated - ZFS_BOOT_PARTITION_SIZE : integer number with `M` or `G` suffix (defaults to `'${c_default_boot_partition_size}M'`) -- ZFS_ENCRYPT_RPOOL : set 1 to encrypt the pool - ZFS_PASSPHRASE : set non-blank to encrypt the pool, and blank not to. if unset, it will be asked. - ZFS_DEBIAN_ROOT_PASSWORD - ZFS_BPOOL_NAME From 24af68c7ece3a09030e606c9d1e02ad86129e371 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 12:12:56 +0200 Subject: [PATCH 02/84] Cosmetic cleanup: Convert apt `-y` to `--yes` --- install-zfs.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index 20e8367..d808caa 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -721,7 +721,7 @@ function install_host_packages_elementary { if [[ ${ZFS_SKIP_LIVE_ZFS_MODULE_INSTALL:-} != "1" ]]; then apt update - apt install -y software-properties-common + apt install --yes software-properties-common fi install_host_packages @@ -752,7 +752,7 @@ function install_host_packages_UbuntuServer { # this will be a no-op. # apt update - apt install -y "linux-headers-$(uname -r)" + apt install --yes "linux-headers-$(uname -r)" install_host_packages else @@ -1156,7 +1156,7 @@ APT' function install_jail_zfs_packages_elementary { print_step_info_header - chroot_execute "apt install -y software-properties-common" + chroot_execute "apt install --yes software-properties-common" install_jail_zfs_packages } From bdf76f7ff89ee59de4abd7e6226ee0d9a350f0ca Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 12:50:06 +0200 Subject: [PATCH 03/84] Cosmetic: Clean variables subdivision Two groups were actually the same concept. --- install-zfs.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index d808caa..443b748 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -12,11 +12,6 @@ set -o nounset # VARIABLES/CONSTANTS ########################################################## -# Variables set by the script - -v_linux_distribution= # Debian, Ubuntu, ... WATCH OUT: not necessarily from `lsb_release` (ie. UbuntuServer) -v_zfs_08_in_repository= # 1=true, false otherwise (applies only to Ubuntu-based) - # Variables set (indirectly) by the user # # The passphrase has a special workflow - it's sent to a named pipe (see create_passphrase_named_pipe()). @@ -38,6 +33,8 @@ v_free_tail_space= # integer # Variables set during execution +v_linux_distribution= # Debian, Ubuntu, ... WATCH OUT: not necessarily from `lsb_release` (ie. UbuntuServer) +v_zfs_08_in_repository= # 1=true, false otherwise (applies only to Ubuntu-based) v_temp_volume_device= # /dev/zdN; scope: setup_partitions -> sync_os_temp_installation_dir_to_rpool v_suitable_disks=() # (/dev/by-id/disk_id, ...); scope: find_suitable_disks -> select_disk From d2a2d9dc7c5c2cf2ba0af7eb5606263bf64ce767 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 12:40:21 +0200 Subject: [PATCH 04/84] Simplification in ZFS package version logic --- install-zfs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-zfs.sh b/install-zfs.sh index 443b748..0d71e2f 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -369,7 +369,7 @@ function find_zfs_package_requirements { apt update local zfs_package_version - zfs_package_version=$(apt show zfsutils-linux 2> /dev/null | perl -ne 'print $1 if /^Version: (\d+\.\d+)\./') + zfs_package_version=$(apt show zfsutils-linux 2> /dev/null | perl -ne 'print /^Version: (\d+\.\d+)/') if [[ -n $zfs_package_version ]]; then if [[ ! $zfs_package_version =~ ^0\. ]]; then From 7dbb663f341cfd5e117fec56249d858ab9c14cd4 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 12:42:40 +0200 Subject: [PATCH 05/84] Use more specific and generic way of checking the ZFS package version This is actually required for allowing OpenZFS v2.0. --- install-zfs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-zfs.sh b/install-zfs.sh index 0d71e2f..b9e171b 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -375,7 +375,7 @@ function find_zfs_package_requirements { if [[ ! $zfs_package_version =~ ^0\. ]]; then >&2 echo "Unsupported ZFS version!: $zfs_package_version" exit 1 - elif (( $(echo "$zfs_package_version" | cut -d. -f2) >= 8 )); then + elif dpkg --compare-versions "$zfs_package_version" ge 0.8; then v_zfs_08_in_repository=1 fi fi From f5ff729128d6d388724728da8ff7531bfb5e8ef5 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 12:45:53 +0200 Subject: [PATCH 06/84] Remove check for unsupported ZFS versions (and simplify logic) 2.0 is going to be supported in this branch, so non-0.x versions must not raise an error. --- install-zfs.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index b9e171b..db28e31 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -371,13 +371,10 @@ function find_zfs_package_requirements { local zfs_package_version zfs_package_version=$(apt show zfsutils-linux 2> /dev/null | perl -ne 'print /^Version: (\d+\.\d+)/') - if [[ -n $zfs_package_version ]]; then - if [[ ! $zfs_package_version =~ ^0\. ]]; then - >&2 echo "Unsupported ZFS version!: $zfs_package_version" - exit 1 - elif dpkg --compare-versions "$zfs_package_version" ge 0.8; then - v_zfs_08_in_repository=1 - fi + # Test returns false if $zfs_package_version is blank. + # + if dpkg --compare-versions "$zfs_package_version" ge 0.8; then + v_zfs_08_in_repository=1 fi } From 9f03180a8e4e9ace96ecb53e6b02e381d0d9c05f Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 12:56:05 +0200 Subject: [PATCH 07/84] Invert ZFS repository logic ("version 0.8 in repo" -> "PPA required") --- install-zfs.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index db28e31..76a68e3 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -34,7 +34,7 @@ v_free_tail_space= # integer # Variables set during execution v_linux_distribution= # Debian, Ubuntu, ... WATCH OUT: not necessarily from `lsb_release` (ie. UbuntuServer) -v_zfs_08_in_repository= # 1=true, false otherwise (applies only to Ubuntu-based) +v_use_ppa= # 1=true, false otherwise (applies only to Ubuntu-based). v_temp_volume_device= # /dev/zdN; scope: setup_partitions -> sync_os_temp_installation_dir_to_rpool v_suitable_disks=() # (/dev/by-id/disk_id, ...); scope: find_suitable_disks -> select_disk @@ -363,7 +363,7 @@ If you think this is a bug, please open an issue on https://github.com/saveriomi # Fortunately, with Debian-specific logic isolated, we need conditionals based only on #2 - see # install_host_packages() and install_host_packages_UbuntuServer(). # -function find_zfs_package_requirements { +function set_zfs_ppa_requirement { print_step_info_header apt update @@ -371,14 +371,14 @@ function find_zfs_package_requirements { local zfs_package_version zfs_package_version=$(apt show zfsutils-linux 2> /dev/null | perl -ne 'print /^Version: (\d+\.\d+)/') - # Test returns false if $zfs_package_version is blank. + # Test returns true if $zfs_package_version is blank. # - if dpkg --compare-versions "$zfs_package_version" ge 0.8; then - v_zfs_08_in_repository=1 + if dpkg --compare-versions "$zfs_package_version" lt 0.8; then + v_use_ppa=1 fi } -function find_zfs_package_requirements_Debian { +function set_zfs_ppa_requirement_Debian { # Only update apt; in this case, ZFS packages are handled in a specific way. apt update @@ -387,12 +387,12 @@ function find_zfs_package_requirements_Debian { # Mint 20 has the CDROM repository enabled, but apt fails when updating due to it (or possibly due # to it being incorrectly setup). # -function find_zfs_package_requirements_Linuxmint { +function set_zfs_ppa_requirement_Linuxmint { print_step_info_header perl -i -pe 's/^(deb cdrom)/# $1/' /etc/apt/sources.list - find_zfs_package_requirements + set_zfs_ppa_requirement } # By using a FIFO, we avoid having to hide statements like `echo $v_passphrase | zpoool create ...` @@ -657,7 +657,7 @@ The option `-O devices=off` is already set, and must not be specified.' function install_host_packages { print_step_info_header - if [[ $v_zfs_08_in_repository != "1" ]]; then + if [[ $v_use_ppa == "1" ]]; then if [[ ${ZFS_SKIP_LIVE_ZFS_MODULE_INSTALL:-} != "1" ]]; then add-apt-repository --yes ppa:jonathonf/zfs apt update @@ -724,7 +724,7 @@ function install_host_packages_elementary { function install_host_packages_UbuntuServer { print_step_info_header - if [[ $v_zfs_08_in_repository == "1" ]]; then + if [[ $v_use_ppa != "1" ]]; then apt install --yes zfsutils-linux efibootmgr zfs --version > "$c_zfs_module_version_log" 2>&1 @@ -1103,7 +1103,7 @@ function prepare_jail { function install_jail_zfs_packages { print_step_info_header - if [[ $v_zfs_08_in_repository != "1" ]]; then + if [[ $v_use_ppa == "1" ]]; then chroot_execute "add-apt-repository --yes ppa:jonathonf/zfs" chroot_execute "apt update" @@ -1158,7 +1158,7 @@ function install_jail_zfs_packages_elementary { function install_jail_zfs_packages_UbuntuServer { print_step_info_header - if [[ $v_zfs_08_in_repository == "1" ]]; then + if [[ $v_use_ppa != "1" ]]; then chroot_execute "apt install --yes zfsutils-linux zfs-initramfs grub-efi-amd64-signed shim-signed" else install_jail_zfs_packages @@ -1421,7 +1421,7 @@ store_running_processes check_prerequisites display_intro_banner find_suitable_disks -distro_dependent_invoke "find_zfs_package_requirements" +distro_dependent_invoke "set_zfs_ppa_requirement" create_passphrase_named_pipe select_disks From 96af7fc06fb9cabcdbe271c7c4896c1d86ac6983 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 13:09:53 +0200 Subject: [PATCH 08/84] Add ZFS_USE_PPA option --- install-zfs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install-zfs.sh b/install-zfs.sh index 76a68e3..9e42cf5 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -166,6 +166,7 @@ This script needs to be run with admin permissions, from a Live CD. The procedure can be entirely automated via environment variables: - ZFS_OS_INSTALLATION_SCRIPT : path of a script to execute instead of Ubiquity (see dedicated section below) +- ZFS_USE_PPA : set to 1 to use packages from `ppa:jonathonf/zfs` (automatically set to true if the O/S version doesn'\''t ship at least v0.8) - ZFS_SELECTED_DISKS : full path of the devices to create the pool on, comma-separated - ZFS_BOOT_PARTITION_SIZE : integer number with `M` or `G` suffix (defaults to `'${c_default_boot_partition_size}M'`) - ZFS_PASSPHRASE : set non-blank to encrypt the pool, and blank not to. if unset, it will be asked. @@ -373,7 +374,7 @@ function set_zfs_ppa_requirement { # Test returns true if $zfs_package_version is blank. # - if dpkg --compare-versions "$zfs_package_version" lt 0.8; then + if [[ ${ZFS_USE_PPA:-} == "1" ]] || dpkg --compare-versions "$zfs_package_version" lt 0.8; then v_use_ppa=1 fi } From dffaefae2f79093a8dc5e6b0da216cb26c6dad86 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 14:40:55 +0200 Subject: [PATCH 09/84] Generalize reference to ZFS make logfile --- install-zfs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-zfs.sh b/install-zfs.sh index 9e42cf5..4976017 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -664,7 +664,7 @@ function install_host_packages { apt update # Libelf-dev allows `CONFIG_STACK_VALIDATION` to be set - it's optional, but good to have. - # Module compilation log: `/var/lib/dkms/zfs/0.8.2/build/make.log` (adjust according to version). + # Module compilation log: `/var/lib/dkms/zfs/**/*/make.log` (adjust according to version). # echo "zfs-dkms zfs-dkms/note-incompatible-licenses note true" | debconf-set-selections apt install --yes libelf-dev zfs-dkms From a47c824c7b8f398215439ec7ba00f1fc406518c5 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 14:42:24 +0200 Subject: [PATCH 10/84] Convert PPA address strings to constant --- install-zfs.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index 4976017..1a236f7 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -43,6 +43,7 @@ v_suitable_disks=() # (/dev/by-id/disk_id, ...); scope: find_suitable_d # Note that Linux Mint is "Linuxmint" from v20 onwards. This actually helps, since some operations are # specific to it. +c_ppa=ppa:jonathonf/zfs c_efi_system_partition_size=512 # megabytes c_default_boot_partition_size=2048 # megabytes c_default_bpool_tweaks="-o ashift=12" @@ -166,7 +167,7 @@ This script needs to be run with admin permissions, from a Live CD. The procedure can be entirely automated via environment variables: - ZFS_OS_INSTALLATION_SCRIPT : path of a script to execute instead of Ubiquity (see dedicated section below) -- ZFS_USE_PPA : set to 1 to use packages from `ppa:jonathonf/zfs` (automatically set to true if the O/S version doesn'\''t ship at least v0.8) +- ZFS_USE_PPA : set to 1 to use packages from `'"$c_ppa"'` (automatically set to true if the O/S version doesn'\''t ship at least v0.8) - ZFS_SELECTED_DISKS : full path of the devices to create the pool on, comma-separated - ZFS_BOOT_PARTITION_SIZE : integer number with `M` or `G` suffix (defaults to `'${c_default_boot_partition_size}M'`) - ZFS_PASSPHRASE : set non-blank to encrypt the pool, and blank not to. if unset, it will be asked. @@ -660,7 +661,7 @@ function install_host_packages { if [[ $v_use_ppa == "1" ]]; then if [[ ${ZFS_SKIP_LIVE_ZFS_MODULE_INSTALL:-} != "1" ]]; then - add-apt-repository --yes ppa:jonathonf/zfs + add-apt-repository --yes "$c_ppa" apt update # Libelf-dev allows `CONFIG_STACK_VALIDATION` to be set - it's optional, but good to have. @@ -1105,7 +1106,7 @@ function install_jail_zfs_packages { print_step_info_header if [[ $v_use_ppa == "1" ]]; then - chroot_execute "add-apt-repository --yes ppa:jonathonf/zfs" + chroot_execute "add-apt-repository --yes $c_ppa" chroot_execute "apt update" From 1b4b776c02d8a5504cc4a4a8434186ebd1d6fa4d Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 17:53:46 +0200 Subject: [PATCH 11/84] Add memory check (when PPA is selected) --- install-zfs.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/install-zfs.sh b/install-zfs.sh index 1a236f7..bb1ca06 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -46,6 +46,7 @@ v_suitable_disks=() # (/dev/by-id/disk_id, ...); scope: find_suitable_d c_ppa=ppa:jonathonf/zfs c_efi_system_partition_size=512 # megabytes c_default_boot_partition_size=2048 # megabytes +c_memory_warning_limit=2880 # megabytes; not set to 3072 because on some systems, some RAM is occupied/shared c_default_bpool_tweaks="-o ashift=12" c_default_rpool_tweaks="-o ashift=12 -O acltype=posixacl -O compression=lz4 -O dnodesize=auto -O relatime=on -O xattr=sa -O normalization=formD" c_zfs_mount_dir=/mnt @@ -377,6 +378,19 @@ function set_zfs_ppa_requirement { # if [[ ${ZFS_USE_PPA:-} == "1" ]] || dpkg --compare-versions "$zfs_package_version" lt 0.8; then v_use_ppa=1 + + local system_memory + system_memory=$(free -m | perl -lane 'print @F[1] if $. == 2') + + if [[ $system_memory -lt $c_memory_warning_limit && -z ${ZFS_NO_INFO_MESSAGES:-} ]]; then + local dialog_message='WARNING! The PPA is used, which requires compiling the ZFS module. + +On systems with relatively little RAM (less than around 3 GiB), the procedure may crash during the compilation. + +In case of crash due to low memory, no error message is displayed; the only traces are `Killed process` messages in the syslog.' + + whiptail --msgbox "$dialog_message" 30 100 + fi fi } From 10f1e6c9cd864b36ad62ef7c0dd6a746ae39dead Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 21:46:53 +0200 Subject: [PATCH 12/84] Remove reference to old guide, and add a comment --- install-zfs.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index bb1ca06..7f51d88 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1002,8 +1002,8 @@ function create_pools { # POOLS CREATION ##################### - # See https://github.com/zfsonlinux/zfs/wiki/Ubuntu-18.04-Root-on-ZFS for the details. - # + # The root pool must be created first, since the boot pool mountpoint is inside it. + # `-R` creates an "Alternate Root Point", which is lost on unmount; it's just a convenience for a temporary mountpoint; # `-f` force overwrite partitions is existing - in some cases, even after wipefs, a filesystem is mistakenly recognized # `-O` set filesystem properties on a pool (pools and filesystems are distincted entities, however, a pool includes an FS by default). @@ -1018,8 +1018,6 @@ function create_pools { "$v_rpool_name" $v_pools_raid_type "${rpool_disks_partitions[@]}" \ < "$c_passphrase_named_pipe" - # `-d` disable all the pool features (not used here); - # # shellcheck disable=SC2086 # TODO: See above zpool create \ "${v_bpool_tweaks[@]}" \ From dcbd0e627be67529bfe9aed335d2283e600a15d0 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 18:49:22 +0200 Subject: [PATCH 13/84] Refactoring: Convert the default pool tweak variables to array --- install-zfs.sh | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index 7f51d88..3107c93 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -47,8 +47,18 @@ c_ppa=ppa:jonathonf/zfs c_efi_system_partition_size=512 # megabytes c_default_boot_partition_size=2048 # megabytes c_memory_warning_limit=2880 # megabytes; not set to 3072 because on some systems, some RAM is occupied/shared -c_default_bpool_tweaks="-o ashift=12" -c_default_rpool_tweaks="-o ashift=12 -O acltype=posixacl -O compression=lz4 -O dnodesize=auto -O relatime=on -O xattr=sa -O normalization=formD" +c_default_bpool_tweaks=( + -o ashift=12 +) +c_default_rpool_tweaks=( + -o ashift=12 + -O acltype=posixacl + -O compression=lz4 + -O dnodesize=auto + -O normalization=formD + -O relatime=on + -O xattr=sa +) c_zfs_mount_dir=/mnt c_installed_os_data_mount_dir=/target declare -A c_supported_linux_distributions=([Debian]=10 [Ubuntu]="18.04 20.04" [UbuntuServer]="18.04 20.04" [LinuxMint]="19.1 19.2 19.3" [Linuxmint]="20 20.1" [elementary]=5.1) @@ -175,8 +185,8 @@ The procedure can be entirely automated via environment variables: - ZFS_DEBIAN_ROOT_PASSWORD - ZFS_BPOOL_NAME - ZFS_RPOOL_NAME -- ZFS_BPOOL_TWEAKS : boot pool options to set on creation (defaults to `'$c_default_bpool_tweaks'`) -- ZFS_RPOOL_TWEAKS : root pool options to set on creation (defaults to `'$c_default_rpool_tweaks'`) +- ZFS_BPOOL_TWEAKS : boot pool options to set on creation (see defaults below) +- ZFS_RPOOL_TWEAKS : root pool options to set on creation (see defaults below) - ZFS_POOLS_RAID_TYPE : options: blank (striping), `mirror`, `raidz`, `raidz2`, `raidz3`; if unset, it will be asked. - ZFS_NO_INFO_MESSAGES : set 1 to skip informational messages - ZFS_SWAP_SIZE : swap size (integer); set 0 for no swap @@ -189,6 +199,10 @@ When installing the O/S via $ZFS_OS_INSTALLATION_SCRIPT, the root pool is mounte 1. the virtual filesystems must be mounted in `'$c_zfs_mount_dir'` (ie. `for vfs in proc sys dev; do mount --rbind /$vfs '$c_zfs_mount_dir'/$vfs; done`) 2. internet must be accessible while chrooting in `'$c_zfs_mount_dir'` (ie. `echo nameserver 8.8.8.8 >> '$c_zfs_mount_dir'/etc/resolv.conf`) 3. `'$c_zfs_mount_dir'` must be left in a dismountable state (e.g. no file locks, no swap etc.); + +Boot pool default tweaks: '"${c_default_bpool_tweaks[@]/#-/$'\n' -}"' + +Root pool default tweaks: '"${c_default_rpool_tweaks[*]/#-/$'\n' -}"' ' echo "$help" @@ -655,7 +669,7 @@ function ask_pool_tweaks { The option `-O devices=off` is already set, and must not be specified.' - local raw_bpool_tweaks=${ZFS_BPOOL_TWEAKS:-$(whiptail --inputbox "$bpool_tweaks_message" 30 100 -- "$c_default_bpool_tweaks" 3>&1 1>&2 2>&3)} + local raw_bpool_tweaks=${ZFS_BPOOL_TWEAKS:-$(whiptail --inputbox "$bpool_tweaks_message" 30 100 -- "${c_default_bpool_tweaks[*]}" 3>&1 1>&2 2>&3)} mapfile -d' ' -t v_bpool_tweaks < <(echo -n "$raw_bpool_tweaks") @@ -663,7 +677,7 @@ The option `-O devices=off` is already set, and must not be specified.' The option `-O devices=off` is already set, and must not be specified.' - local raw_rpool_tweaks=${ZFS_RPOOL_TWEAKS:-$(whiptail --inputbox "$rpool_tweaks_message" 30 100 -- "$c_default_rpool_tweaks" 3>&1 1>&2 2>&3)} + local raw_rpool_tweaks=${ZFS_RPOOL_TWEAKS:-$(whiptail --inputbox "$rpool_tweaks_message" 30 100 -- "${c_default_rpool_tweaks[*]}" 3>&1 1>&2 2>&3)} mapfile -d' ' -t v_rpool_tweaks < <(echo -n "$raw_rpool_tweaks") From c8d35646f1cc1aef72e46a1f753fe5afc69a311f Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 19:15:06 +0200 Subject: [PATCH 14/84] Refactoring: Rename the "tweaks" concept to "create options", and consolidate them This is in preparation for the procedure update. --- install-zfs.sh | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index 3107c93..b2cd838 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -22,10 +22,10 @@ set -o nounset v_boot_partition_size= # Integer number with `M` or `G` suffix v_bpool_name= -v_bpool_tweaks= # array; see defaults below for format +v_bpool_create_options= # array; see defaults below for format v_root_password= # Debian-only v_rpool_name= -v_rpool_tweaks= # array; see defaults below for format +v_rpool_create_options= # array; see defaults below for format v_pools_raid_type= declare -a v_selected_disks # (/dev/by-id/disk_id, ...) v_swap_size= # integer @@ -47,10 +47,11 @@ c_ppa=ppa:jonathonf/zfs c_efi_system_partition_size=512 # megabytes c_default_boot_partition_size=2048 # megabytes c_memory_warning_limit=2880 # megabytes; not set to 3072 because on some systems, some RAM is occupied/shared -c_default_bpool_tweaks=( +c_default_bpool_create_options=( -o ashift=12 + -O devices=off ) -c_default_rpool_tweaks=( +c_default_rpool_create_options=( -o ashift=12 -O acltype=posixacl -O compression=lz4 @@ -58,6 +59,7 @@ c_default_rpool_tweaks=( -O normalization=formD -O relatime=on -O xattr=sa + -O devices=off ) c_zfs_mount_dir=/mnt c_installed_os_data_mount_dir=/target @@ -185,8 +187,8 @@ The procedure can be entirely automated via environment variables: - ZFS_DEBIAN_ROOT_PASSWORD - ZFS_BPOOL_NAME - ZFS_RPOOL_NAME -- ZFS_BPOOL_TWEAKS : boot pool options to set on creation (see defaults below) -- ZFS_RPOOL_TWEAKS : root pool options to set on creation (see defaults below) +- ZFS_BPOOL_CREATE_OPTIONS : boot pool options to set on creation (see defaults below) +- ZFS_RPOOL_CREATE_OPTIONS : root pool options to set on creation (see defaults below) - ZFS_POOLS_RAID_TYPE : options: blank (striping), `mirror`, `raidz`, `raidz2`, `raidz3`; if unset, it will be asked. - ZFS_NO_INFO_MESSAGES : set 1 to skip informational messages - ZFS_SWAP_SIZE : swap size (integer); set 0 for no swap @@ -200,9 +202,9 @@ When installing the O/S via $ZFS_OS_INSTALLATION_SCRIPT, the root pool is mounte 2. internet must be accessible while chrooting in `'$c_zfs_mount_dir'` (ie. `echo nameserver 8.8.8.8 >> '$c_zfs_mount_dir'/etc/resolv.conf`) 3. `'$c_zfs_mount_dir'` must be left in a dismountable state (e.g. no file locks, no swap etc.); -Boot pool default tweaks: '"${c_default_bpool_tweaks[@]/#-/$'\n' -}"' +Boot pool default create options: '"${c_default_bpool_create_options[*]/#-/$'\n' -}"' -Root pool default tweaks: '"${c_default_rpool_tweaks[*]/#-/$'\n' -}"' +Root pool default create options: '"${c_default_rpool_create_options[*]/#-/$'\n' -}"' ' echo "$help" @@ -662,26 +664,26 @@ function ask_pool_names { print_variables v_bpool_name v_rpool_name } -function ask_pool_tweaks { +function ask_pool_create_options { print_step_info_header - local bpool_tweaks_message='Insert the tweaks for the boot pool + local bpool_create_options_message='Insert the create options for the boot pool -The option `-O devices=off` is already set, and must not be specified.' +The mount-related options are automatically added, and must not be specified.' - local raw_bpool_tweaks=${ZFS_BPOOL_TWEAKS:-$(whiptail --inputbox "$bpool_tweaks_message" 30 100 -- "${c_default_bpool_tweaks[*]}" 3>&1 1>&2 2>&3)} + local raw_bpool_create_options=${ZFS_BPOOL_CREATE_OPTIONS:-$(whiptail --inputbox "$bpool_create_options_message" 30 100 -- "${c_default_bpool_create_options[*]}" 3>&1 1>&2 2>&3)} - mapfile -d' ' -t v_bpool_tweaks < <(echo -n "$raw_bpool_tweaks") + mapfile -d' ' -t v_bpool_create_options < <(echo -n "$raw_bpool_create_options") - local rpool_tweaks_message='Insert the tweaks for the root pool + local rpool_create_options_message='Insert the create options for the root pool -The option `-O devices=off` is already set, and must not be specified.' +The encryption/mount-related options are automatically added, and must not be specified.' - local raw_rpool_tweaks=${ZFS_RPOOL_TWEAKS:-$(whiptail --inputbox "$rpool_tweaks_message" 30 100 -- "${c_default_rpool_tweaks[*]}" 3>&1 1>&2 2>&3)} + local raw_rpool_create_options=${ZFS_RPOOL_CREATE_OPTIONS:-$(whiptail --inputbox "$rpool_create_options_message" 30 100 -- "${c_default_rpool_create_options[*]}" 3>&1 1>&2 2>&3)} - mapfile -d' ' -t v_rpool_tweaks < <(echo -n "$raw_rpool_tweaks") + mapfile -d' ' -t v_rpool_create_options < <(echo -n "$raw_rpool_create_options") - print_variables v_bpool_tweaks v_rpool_tweaks + print_variables v_bpool_create_options v_rpool_create_options } function install_host_packages { @@ -1027,15 +1029,15 @@ function create_pools { # shellcheck disable=SC2086 # TODO: convert v_pools_raid_type to array, and quote zpool create \ "${encryption_options[@]}" \ - "${v_rpool_tweaks[@]}" \ - -O devices=off -O mountpoint=/ -R "$c_zfs_mount_dir" -f \ + "${v_rpool_create_options[@]}" \ + -O mountpoint=/ -R "$c_zfs_mount_dir" -f \ "$v_rpool_name" $v_pools_raid_type "${rpool_disks_partitions[@]}" \ < "$c_passphrase_named_pipe" # shellcheck disable=SC2086 # TODO: See above zpool create \ - "${v_bpool_tweaks[@]}" \ - -O devices=off -O mountpoint=/boot -R "$c_zfs_mount_dir" -f \ + "${v_bpool_create_options[@]}" \ + -O mountpoint=/boot -R "$c_zfs_mount_dir" -f \ "$v_bpool_name" $v_pools_raid_type "${bpool_disks_partitions[@]}" } @@ -1460,7 +1462,7 @@ ask_boot_partition_size ask_swap_size ask_free_tail_space ask_pool_names -ask_pool_tweaks +ask_pool_create_options distro_dependent_invoke "install_host_packages" setup_partitions From 489099a7289bd50926be831f645e6e7dcb567726 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 22:35:08 +0200 Subject: [PATCH 15/84] Split install_and_configure_bootloader in separate functions --- install-zfs.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index b2cd838..42a2f72 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1195,7 +1195,7 @@ function install_jail_zfs_packages_UbuntuServer { fi } -function install_and_configure_bootloader { +function prepare_efi_partition { print_step_info_header chroot_execute "echo PARTUUID=$(blkid -s PARTUUID -o value "${v_selected_disks[0]}-part1") /boot/efi vfat nofail,x-systemd.device-timeout=1 0 1 > /etc/fstab" @@ -1204,6 +1204,10 @@ function install_and_configure_bootloader { chroot_execute "mount /boot/efi" chroot_execute "grub-install" +} + +function configure_and_update_grub { + print_step_info_header chroot_execute "perl -i -pe 's/(GRUB_CMDLINE_LINUX=\")/\${1}root=ZFS=$v_rpool_name /' /etc/default/grub" @@ -1230,16 +1234,9 @@ function install_and_configure_bootloader { chroot_execute "update-grub" } -function install_and_configure_bootloader_Debian { +function configure_and_update_grub_Debian { print_step_info_header - chroot_execute "echo PARTUUID=$(blkid -s PARTUUID -o value "${v_selected_disks[0]}-part1") /boot/efi vfat nofail,x-systemd.device-timeout=1 0 1 > /etc/fstab" - - chroot_execute "mkdir -p /boot/efi" - chroot_execute "mount /boot/efi" - - chroot_execute "grub-install" - chroot_execute "perl -i -pe 's/(GRUB_CMDLINE_LINUX=\")/\${1}root=ZFS=$v_rpool_name /' /etc/default/grub" chroot_execute "perl -i -pe 's/(GRUB_CMDLINE_LINUX_DEFAULT=.*)quiet/\$1/' /etc/default/grub" chroot_execute "perl -i -pe 's/#(GRUB_TERMINAL=console)/\$1/' /etc/default/grub" @@ -1485,7 +1482,8 @@ fi prepare_jail distro_dependent_invoke "install_jail_zfs_packages" -distro_dependent_invoke "install_and_configure_bootloader" +prepare_efi_partition +distro_dependent_invoke "configure_and_update_grub" sync_efi_partitions configure_boot_pool_import update_initramfs From 65997e33ad74ff0b711db2cbf5ac61e7bd36cf70 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 22:46:05 +0200 Subject: [PATCH 16/84] Add GRUB option `init_on_alloc` to address performance regression --- install-zfs.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/install-zfs.sh b/install-zfs.sh index 42a2f72..295e61b 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1209,6 +1209,8 @@ function prepare_efi_partition { function configure_and_update_grub { print_step_info_header + chroot_execute "perl -i -pe 's/GRUB_CMDLINE_LINUX_DEFAULT=\"\K/init_on_alloc=0 /' /etc/default/grub" + chroot_execute "perl -i -pe 's/(GRUB_CMDLINE_LINUX=\")/\${1}root=ZFS=$v_rpool_name /' /etc/default/grub" # Silence warning during the grub probe (source: https://git.io/JenXF). @@ -1237,6 +1239,8 @@ function configure_and_update_grub { function configure_and_update_grub_Debian { print_step_info_header + chroot_execute "perl -i -pe 's/GRUB_CMDLINE_LINUX_DEFAULT=\"\K/init_on_alloc=0 /' /etc/default/grub" + chroot_execute "perl -i -pe 's/(GRUB_CMDLINE_LINUX=\")/\${1}root=ZFS=$v_rpool_name /' /etc/default/grub" chroot_execute "perl -i -pe 's/(GRUB_CMDLINE_LINUX_DEFAULT=.*)quiet/\$1/' /etc/default/grub" chroot_execute "perl -i -pe 's/#(GRUB_TERMINAL=console)/\$1/' /etc/default/grub" From c256f4dad304c54e5c91559bfb984a1168f7250f Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 22:58:37 +0200 Subject: [PATCH 17/84] Disable bpool name customization This is mandated by the updated procedure. --- README.md | 1 - install-zfs.sh | 37 ++++++++++++------------------------- 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 6654515..a585b74 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,6 @@ The procedure can be entirely automated via environment variables: - ZFS_OS_INSTALLATION_SCRIPT : path of a script to execute instead of Ubiquity (see dedicated section below) - ZFS_SELECTED_DISKS : full path of the devices to create the pool on, comma-separated - ZFS_PASSPHRASE -- ZFS_BPOOL_NAME - ZFS_RPOOL_NAME - ZFS_BPOOL_TWEAKS : boot pool options to set on creation (defaults to `-o ashift=12`) - ZFS_RPOOL_TWEAKS : root pool options to set on creation (defaults to `-o ashift=12 -O acltype=posixacl -O compression=lz4 -O dnodesize=auto -O relatime=on -O xattr=sa -O normalization=formD`) diff --git a/install-zfs.sh b/install-zfs.sh index 295e61b..9754a4a 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -21,7 +21,6 @@ set -o nounset # Note that `ZFS_PASSPHRASE` and `ZFS_POOLS_RAID_TYPE` consider the unset state (see help). v_boot_partition_size= # Integer number with `M` or `G` suffix -v_bpool_name= v_bpool_create_options= # array; see defaults below for format v_root_password= # Debian-only v_rpool_name= @@ -43,6 +42,7 @@ v_suitable_disks=() # (/dev/by-id/disk_id, ...); scope: find_suitable_d # Note that Linux Mint is "Linuxmint" from v20 onwards. This actually helps, since some operations are # specific to it. +c_bpool_name=bpool c_ppa=ppa:jonathonf/zfs c_efi_system_partition_size=512 # megabytes c_default_boot_partition_size=2048 # megabytes @@ -185,7 +185,6 @@ The procedure can be entirely automated via environment variables: - ZFS_BOOT_PARTITION_SIZE : integer number with `M` or `G` suffix (defaults to `'${c_default_boot_partition_size}M'`) - ZFS_PASSPHRASE : set non-blank to encrypt the pool, and blank not to. if unset, it will be asked. - ZFS_DEBIAN_ROOT_PASSWORD -- ZFS_BPOOL_NAME - ZFS_RPOOL_NAME - ZFS_BPOOL_CREATE_OPTIONS : boot pool options to set on creation (see defaults below) - ZFS_RPOOL_CREATE_OPTIONS : root pool options to set on creation (see defaults below) @@ -634,21 +633,9 @@ For detailed informations, see the wiki page: https://github.com/saveriomiroddi/ print_variables v_free_tail_space } -function ask_pool_names { +function ask_rpool_name { print_step_info_header - if [[ ${ZFS_BPOOL_NAME:-} != "" ]]; then - v_bpool_name=$ZFS_BPOOL_NAME - else - local bpool_name_invalid_message= - - while [[ ! $v_bpool_name =~ ^[a-z][a-zA-Z_:.-]+$ ]]; do - v_bpool_name=$(whiptail --inputbox "${bpool_name_invalid_message}Insert the name for the boot pool" 30 100 bpool 3>&1 1>&2 2>&3) - - bpool_name_invalid_message="Invalid pool name! " - done - fi - if [[ ${ZFS_RPOOL_NAME:-} != "" ]]; then v_rpool_name=$ZFS_RPOOL_NAME else @@ -661,7 +648,7 @@ function ask_pool_names { done fi - print_variables v_bpool_name v_rpool_name + print_variables v_rpool_name } function ask_pool_create_options { @@ -1038,7 +1025,7 @@ function create_pools { zpool create \ "${v_bpool_create_options[@]}" \ -O mountpoint=/boot -R "$c_zfs_mount_dir" -f \ - "$v_bpool_name" $v_pools_raid_type "${bpool_disks_partitions[@]}" + "$c_bpool_name" $v_pools_raid_type "${bpool_disks_partitions[@]}" } function create_swap_volume { @@ -1106,7 +1093,7 @@ function remove_temp_partition_and_expand_rpool { # For unencrypted pools, `-l` doesn't interfere. # zpool import -l -R "$c_zfs_mount_dir" "$v_rpool_name" < "$c_passphrase_named_pipe_2" - zpool import -l -R "$c_zfs_mount_dir" "$v_bpool_name" + zpool import -l -R "$c_zfs_mount_dir" "$c_bpool_name" for selected_disk in "${v_selected_disks[@]}"; do zpool online -e "$v_rpool_name" "$selected_disk-part3" @@ -1272,7 +1259,7 @@ function sync_efi_partitions { function configure_boot_pool_import { print_step_info_header - chroot_execute "cat > /etc/systemd/system/zfs-import-$v_bpool_name.service < /etc/systemd/system/zfs-import-$c_bpool_name.service <> /etc/fstab" + chroot_execute "zfs set mountpoint=legacy $c_bpool_name" + chroot_execute "echo $c_bpool_name /boot zfs nodev,relatime,x-systemd.requires=zfs-import-$c_bpool_name.service 0 0 >> /etc/fstab" } # This step is important in cases where the keyboard layout is not the standard one. @@ -1364,7 +1351,7 @@ ConditionVirtualization=!container [Service] Type=oneshot -ExecStart=/sbin/zpool trim $v_bpool_name +ExecStart=/sbin/zpool trim $c_bpool_name ExecStart=/sbin/zpool trim $v_rpool_name UNIT" @@ -1462,7 +1449,7 @@ ask_encryption ask_boot_partition_size ask_swap_size ask_free_tail_space -ask_pool_names +ask_rpool_name ask_pool_create_options distro_dependent_invoke "install_host_packages" From 20df89306ee3eb5dc7948badb0379c860bebd23a Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 25 Apr 2021 19:48:14 +0200 Subject: [PATCH 18/84] Cosmetic fixes --- install-zfs.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index 9754a4a..fc600c0 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -323,7 +323,7 @@ function find_suitable_disks { candidate_disk_ids=$(find /dev/disk/by-id -regextype awk -regex '.+/(ata|nvme|scsi|mmc)-.+' -not -regex '.+-part[0-9]+$' | sort) mounted_devices="$(df | awk 'BEGIN {getline} {print $1}' | xargs -n 1 lsblk -no pkname 2> /dev/null | sort -u || true)" - while read -r disk_id || [[ -n "$disk_id" ]]; do + while read -r disk_id || [[ -n $disk_id ]]; do local device_info local block_device_basename @@ -351,7 +351,6 @@ function find_suitable_disks { $(udevadm info --query=property "$(readlink -f "$disk_id")") LOG - done < <(echo -n "$candidate_disk_ids") if [[ ${#v_suitable_disks[@]} -eq 0 ]]; then From 13b3f580217f79f0b4285232b68a57c487693200 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 25 Apr 2021 20:21:09 +0200 Subject: [PATCH 19/84] Sort disks in selection dialog --- install-zfs.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/install-zfs.sh b/install-zfs.sh index fc600c0..aecf694 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -452,11 +452,16 @@ function select_disks { local disk_selection_status=OFF fi + # St00pid simple way of sorting by block device name. Relies on the tokens not including whitespace. + for disk_id in "${v_suitable_disks[@]}"; do block_device_basename="$(basename "$(readlink -f "$disk_id")")" - menu_entries_option+=("$disk_id" "($block_device_basename)" "$disk_selection_status") + menu_entries_option+=("$disk_id ($block_device_basename) $disk_selection_status") done + # shellcheck disable=2207 # cheating here, for simplicity (alternative: add tr and mapfile). + menu_entries_option=($(printf $'%s\n' "${menu_entries_option[@]}" | sort -k 2)) + local dialog_message="Select the ZFS devices. Devices with mounted partitions, cdroms, and removable devices are not displayed! From c7698cffdca17e5222cacf4cf2cbecc55cde0c8a Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 25 Apr 2021 20:36:59 +0200 Subject: [PATCH 20/84] Update memory warning message and placement For simplicity, always display it (when the memory is not enough). Debian ended to be a real headscratcher, so it's best to be very clear. --- install-zfs.sh | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index aecf694..d7920fb 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -46,7 +46,7 @@ c_bpool_name=bpool c_ppa=ppa:jonathonf/zfs c_efi_system_partition_size=512 # megabytes c_default_boot_partition_size=2048 # megabytes -c_memory_warning_limit=2880 # megabytes; not set to 3072 because on some systems, some RAM is occupied/shared +c_memory_warning_limit=$((3584 - 128)) # megabytes; exclude some RAM, which can be occupied/shared c_default_bpool_create_options=( -o ashift=12 -O devices=off @@ -303,6 +303,24 @@ In order to stop the procedure, hit Esc twice during dialogs (excluding yes/no o fi } +function check_system_memory { + local system_memory + system_memory=$(free -m | perl -lane 'print @F[1] if $. == 2') + + if [[ $system_memory -lt $c_memory_warning_limit && -z ${ZFS_NO_INFO_MESSAGES:-} ]]; then + # A workaround for these cases is to use the swap generate, but this can potentially cause troubles + # (severe compilation slowdowns) if a user tries to compensate too little memory with a large swapfile. + # + local dialog_message='WARNING! In some cases, the ZFS modules require compilation. + +On systems with relatively little RAM, the procedure may crash during the compilation, for example with 3 GB on Debian 10.9. + +In such cases, the module building may fail abruptly, either without visible errors (leaving "process killed" messages in the syslog), or with package installation errors (leaving odd errors in the module'\''s `make.log`).' + + whiptail --msgbox "$dialog_message" 30 100 + fi +} + function find_suitable_disks { print_step_info_header @@ -392,19 +410,6 @@ function set_zfs_ppa_requirement { # if [[ ${ZFS_USE_PPA:-} == "1" ]] || dpkg --compare-versions "$zfs_package_version" lt 0.8; then v_use_ppa=1 - - local system_memory - system_memory=$(free -m | perl -lane 'print @F[1] if $. == 2') - - if [[ $system_memory -lt $c_memory_warning_limit && -z ${ZFS_NO_INFO_MESSAGES:-} ]]; then - local dialog_message='WARNING! The PPA is used, which requires compiling the ZFS module. - -On systems with relatively little RAM (less than around 3 GiB), the procedure may crash during the compilation. - -In case of crash due to low memory, no error message is displayed; the only traces are `Killed process` messages in the syslog.' - - whiptail --msgbox "$dialog_message" 30 100 - fi fi } @@ -1442,6 +1447,7 @@ distro_dependent_invoke "store_os_distro_information" store_running_processes check_prerequisites display_intro_banner +check_system_memory find_suitable_disks distro_dependent_invoke "set_zfs_ppa_requirement" create_passphrase_named_pipe From ae7f059406db9198b7beab5a463b20612d4e2ff9 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 25 Apr 2021 20:46:10 +0200 Subject: [PATCH 21/84] Clean pools raid type option, using a Bash array --- install-zfs.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index d7920fb..bbc6172 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -25,7 +25,7 @@ v_bpool_create_options= # array; see defaults below for format v_root_password= # Debian-only v_rpool_name= v_rpool_create_options= # array; see defaults below for format -v_pools_raid_type= +v_pools_raid_type=() declare -a v_selected_disks # (/dev/by-id/disk_id, ...) v_swap_size= # integer v_free_tail_space= # integer @@ -485,8 +485,10 @@ Devices with mounted partitions, cdroms, and removable devices are not displayed function select_pools_raid_type { print_step_info_header + local raw_pools_raid_type= + if [[ -v ZFS_POOLS_RAID_TYPE ]]; then - v_pools_raid_type=$ZFS_POOLS_RAID_TYPE + raw_pools_raid_type=$ZFS_POOLS_RAID_TYPE elif [[ ${#v_selected_disks[@]} -ge 2 ]]; then # Entries preparation. @@ -517,7 +519,11 @@ function select_pools_raid_type { fi local dialog_message="Select the pools RAID type." - v_pools_raid_type=$(whiptail --radiolist "$dialog_message" 30 100 $((${#menu_entries_option[@]} / 3)) "${menu_entries_option[@]}" 3>&1 1>&2 2>&3) + raw_pools_raid_type=$(whiptail --radiolist "$dialog_message" 30 100 $((${#menu_entries_option[@]} / 3)) "${menu_entries_option[@]}" 3>&1 1>&2 2>&3) + fi + + if [[ -n $raw_pools_raid_type ]]; then + v_pools_raid_type=("$raw_pools_raid_type") fi } @@ -1022,19 +1028,17 @@ function create_pools { # # Stdin is ignored if the encryption is not set (and set via prompt). # - # shellcheck disable=SC2086 # TODO: convert v_pools_raid_type to array, and quote zpool create \ "${encryption_options[@]}" \ "${v_rpool_create_options[@]}" \ -O mountpoint=/ -R "$c_zfs_mount_dir" -f \ - "$v_rpool_name" $v_pools_raid_type "${rpool_disks_partitions[@]}" \ + "$v_rpool_name" "${v_pools_raid_type[@]}" "${rpool_disks_partitions[@]}" \ < "$c_passphrase_named_pipe" - # shellcheck disable=SC2086 # TODO: See above zpool create \ "${v_bpool_create_options[@]}" \ -O mountpoint=/boot -R "$c_zfs_mount_dir" -f \ - "$c_bpool_name" $v_pools_raid_type "${bpool_disks_partitions[@]}" + "$c_bpool_name" "${v_pools_raid_type[@]}" "${bpool_disks_partitions[@]}" } function create_swap_volume { From 0d650085c5195016b608e914b560cbdaafde3ebf Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 25 Apr 2021 21:20:08 +0200 Subject: [PATCH 22/84] Print internal variables as ZFS_* exports This makes debugging speedier. --- install-zfs.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/install-zfs.sh b/install-zfs.sh index bbc6172..f949e6a 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -441,6 +441,31 @@ function create_passphrase_named_pipe { mkfifo "$c_passphrase_named_pipe" "$c_passphrase_named_pipe_2" } +function register_exit_hook { + function _exit_hook { + # Only the meaningful variable(s) are printed. + # In order to print the password, the store strategy should be changed, as the pipes may be empty. + # + echo "\ +# Currently set exports, for performing an unattended (as possible) installation with the same configuration: +# +export ZFS_USE_PPA=$v_use_ppa +export ZFS_SELECTED_DISKS=$(IFS=,; echo -n "${v_selected_disks[*]}") +export ZFS_BOOT_PARTITION_SIZE=$v_boot_partition_size +export ZFS_PASSPHRASE=_currently_not_available_ +export ZFS_DEBIAN_ROOT_PASSWORD=$(printf %q "$v_root_password") +export ZFS_RPOOL_NAME=$v_rpool_name +export ZFS_BPOOL_CREATE_OPTIONS=\"${v_bpool_create_options[*]}\" +export ZFS_RPOOL_CREATE_OPTIONS=\"${v_bpool_create_options[*]}\" +export ZFS_POOLS_RAID_TYPE=${v_pools_raid_type[*]} +export ZFS_NO_INFO_MESSAGES=${ZFS_NO_INFO_MESSAGES:-} +export ZFS_SWAP_SIZE=$v_swap_size +export ZFS_FREE_TAIL_SPACE=$v_free_tail_space +" + } + trap _exit_hook EXIT +} + function select_disks { print_step_info_header @@ -1455,6 +1480,7 @@ check_system_memory find_suitable_disks distro_dependent_invoke "set_zfs_ppa_requirement" create_passphrase_named_pipe +register_exit_hook select_disks select_pools_raid_type From 3b4803063b7dde7157dac6ce2235d0de1c8b8558 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 25 Apr 2021 21:24:27 +0200 Subject: [PATCH 23/84] Remove named pipe files on exit Now that there's an exit hook, we can cleanup after ourselves :). --- install-zfs.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index f949e6a..fb141aa 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -433,16 +433,14 @@ function set_zfs_ppa_requirement_Linuxmint { # By using a FIFO, we avoid having to hide statements like `echo $v_passphrase | zpoool create ...` # from the logs. # -# The FIFO file is left in the filesystem after the script exits. It's not worth taking care of -# removing it, since the environment is entirely ephemeral. -# function create_passphrase_named_pipe { - rm -f "$c_passphrase_named_pipe" "$c_passphrase_named_pipe_2" mkfifo "$c_passphrase_named_pipe" "$c_passphrase_named_pipe_2" } function register_exit_hook { function _exit_hook { + rm -f "$c_passphrase_named_pipe" "$c_passphrase_named_pipe_2" + # Only the meaningful variable(s) are printed. # In order to print the password, the store strategy should be changed, as the pipes may be empty. # @@ -1479,8 +1477,8 @@ display_intro_banner check_system_memory find_suitable_disks distro_dependent_invoke "set_zfs_ppa_requirement" -create_passphrase_named_pipe register_exit_hook +create_passphrase_named_pipe select_disks select_pools_raid_type From d9f2cce4b4a3411da64dc99c0df994d3ca56e9af Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 25 Apr 2021 21:33:46 +0200 Subject: [PATCH 24/84] Simplify passphrase internal handling --- install-zfs.sh | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index fb141aa..d90f369 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -22,6 +22,7 @@ set -o nounset v_boot_partition_size= # Integer number with `M` or `G` suffix v_bpool_create_options= # array; see defaults below for format +v_passphrase= v_root_password= # Debian-only v_rpool_name= v_rpool_create_options= # array; see defaults below for format @@ -66,7 +67,6 @@ c_installed_os_data_mount_dir=/target declare -A c_supported_linux_distributions=([Debian]=10 [Ubuntu]="18.04 20.04" [UbuntuServer]="18.04 20.04" [LinuxMint]="19.1 19.2 19.3" [Linuxmint]="20 20.1" [elementary]=5.1) c_temporary_volume_size=12 # gigabytes; large enough - Debian, for example, takes ~8 GiB. c_passphrase_named_pipe=$(dirname "$(mktemp)")/zfs-installer.pp.fifo -c_passphrase_named_pipe_2=$(dirname "$(mktemp)")/zfs-installer.pp.2.fifo c_log_dir=$(dirname "$(mktemp)")/zfs-installer c_install_log=$c_log_dir/install.log @@ -434,12 +434,12 @@ function set_zfs_ppa_requirement_Linuxmint { # from the logs. # function create_passphrase_named_pipe { - mkfifo "$c_passphrase_named_pipe" "$c_passphrase_named_pipe_2" + mkfifo "$c_passphrase_named_pipe" } function register_exit_hook { function _exit_hook { - rm -f "$c_passphrase_named_pipe" "$c_passphrase_named_pipe_2" + rm -f "$c_passphrase_named_pipe" # Only the meaningful variable(s) are printed. # In order to print the password, the store strategy should be changed, as the pipes may be empty. @@ -573,25 +573,23 @@ function ask_root_password_Debian { function ask_encryption { print_step_info_header - local passphrase= - set +x if [[ -v ZFS_PASSPHRASE ]]; then - passphrase=$ZFS_PASSPHRASE + v_passphrase=$ZFS_PASSPHRASE else local passphrase_repeat=_ local passphrase_invalid_message= - while [[ $passphrase != "$passphrase_repeat" || ${#passphrase} -lt 8 ]]; do + while [[ $v_passphrase != "$passphrase_repeat" || ${#v_passphrase} -lt 8 ]]; do local dialog_message="${passphrase_invalid_message}Please enter the passphrase (8 chars min.): Leave blank to keep encryption disabled. " - passphrase=$(whiptail --passwordbox "$dialog_message" 30 100 3>&1 1>&2 2>&3) + v_passphrase=$(whiptail --passwordbox "$dialog_message" 30 100 3>&1 1>&2 2>&3) - if [[ -z $passphrase ]]; then + if [[ -z $v_passphrase ]]; then break fi @@ -601,9 +599,6 @@ Leave blank to keep encryption disabled. done fi - echo -n "$passphrase" > "$c_passphrase_named_pipe" & - echo -n "$passphrase" > "$c_passphrase_named_pipe_2" & - set -x } @@ -1017,23 +1012,14 @@ function custom_install_operating_system { function create_pools { # POOL OPTIONS ####################### - local passphrase local encryption_options=() local rpool_disks_partitions=() local bpool_disks_partitions=() set +x - - passphrase=$(cat "$c_passphrase_named_pipe") - - if [[ -n $passphrase ]]; then + if [[ -n $v_passphrase ]]; then encryption_options=(-O "encryption=on" -O "keylocation=prompt" -O "keyformat=passphrase") fi - - # Push back for unlogged reuse. Minor inconvenience, but worth :-) - # - echo -n "$passphrase" > "$c_passphrase_named_pipe" & - set -x for selected_disk in "${v_selected_disks[@]}"; do @@ -1045,6 +1031,10 @@ function create_pools { # The root pool must be created first, since the boot pool mountpoint is inside it. + set +x + echo -n "$v_passphrase" > "$c_passphrase_named_pipe" & + set -x + # `-R` creates an "Alternate Root Point", which is lost on unmount; it's just a convenience for a temporary mountpoint; # `-f` force overwrite partitions is existing - in some cases, even after wipefs, a filesystem is mistakenly recognized # `-O` set filesystem properties on a pool (pools and filesystems are distincted entities, however, a pool includes an FS by default). @@ -1126,9 +1116,13 @@ function remove_temp_partition_and_expand_rpool { parted -s "$selected_disk" unit s resizepart 3 -- "$resize_reference" done + set +x + echo -n "$v_passphrase" > "$c_passphrase_named_pipe" & + set -x + # For unencrypted pools, `-l` doesn't interfere. # - zpool import -l -R "$c_zfs_mount_dir" "$v_rpool_name" < "$c_passphrase_named_pipe_2" + zpool import -l -R "$c_zfs_mount_dir" "$v_rpool_name" < "$c_passphrase_named_pipe" zpool import -l -R "$c_zfs_mount_dir" "$c_bpool_name" for selected_disk in "${v_selected_disks[@]}"; do From 1abf9f10fdb81f8cf2c5ba27255e135f46e9a3f4 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 25 Apr 2021 21:34:29 +0200 Subject: [PATCH 25/84] Improvements and fix to exports printing --- install-zfs.sh | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index d90f369..fe50929 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -441,25 +441,45 @@ function register_exit_hook { function _exit_hook { rm -f "$c_passphrase_named_pipe" + set +x + # Only the meaningful variable(s) are printed. # In order to print the password, the store strategy should be changed, as the pipes may be empty. # - echo "\ -# Currently set exports, for performing an unattended (as possible) installation with the same configuration: -# + echo " +Currently set exports, for performing an unattended (as possible) installation with the same configuration: + export ZFS_USE_PPA=$v_use_ppa export ZFS_SELECTED_DISKS=$(IFS=,; echo -n "${v_selected_disks[*]}") export ZFS_BOOT_PARTITION_SIZE=$v_boot_partition_size -export ZFS_PASSPHRASE=_currently_not_available_ +export ZFS_PASSPHRASE=$(printf %q "$v_passphrase") export ZFS_DEBIAN_ROOT_PASSWORD=$(printf %q "$v_root_password") export ZFS_RPOOL_NAME=$v_rpool_name export ZFS_BPOOL_CREATE_OPTIONS=\"${v_bpool_create_options[*]}\" -export ZFS_RPOOL_CREATE_OPTIONS=\"${v_bpool_create_options[*]}\" +export ZFS_RPOOL_CREATE_OPTIONS=\"${v_rpool_create_options[*]}\" export ZFS_POOLS_RAID_TYPE=${v_pools_raid_type[*]} -export ZFS_NO_INFO_MESSAGES=${ZFS_NO_INFO_MESSAGES:-} +export ZFS_NO_INFO_MESSAGES=1 export ZFS_SWAP_SIZE=$v_swap_size -export ZFS_FREE_TAIL_SPACE=$v_free_tail_space -" +export ZFS_FREE_TAIL_SPACE=$v_free_tail_space" + + # Convenient ready exports (selecting the first two disks): + # + local ready=" +export ZFS_USE_PPA= +export ZFS_SELECTED_DISKS=$(ls -l /dev/disk/by-id/ | perl -ane 'print "/dev/disk/by-id/@F[8]," if ! /\d$/ && ($c += 1) <= 2' | head -c -1) +export ZFS_BOOT_PARTITION_SIZE=2048M +export ZFS_PASSPHRASE=aaaaaaaa +export ZFS_DEBIAN_ROOT_PASSWORD=a +export ZFS_RPOOL_NAME=rpool +export ZFS_BPOOL_CREATE_OPTIONS='-o ashift=12 -o autotrim=on -d -o feature@async_destroy=enabled -o feature@bookmarks=enabled -o feature@embedded_data=enabled -o feature@empty_bpobj=enabled -o feature@enabled_txg=enabled -o feature@extensible_dataset=enabled -o feature@filesystem_limits=enabled -o feature@hole_birth=enabled -o feature@large_blocks=enabled -o feature@lz4_compress=enabled -o feature@spacemap_histogram=enabled -O acltype=posixacl -O compression=lz4 -O devices=off -O normalization=formD -O relatime=on -O xattr=sa' +export ZFS_RPOOL_CREATE_OPTIONS='-o ashift=12 -o autotrim=on -O acltype=posixacl -O compression=lz4 -O dnodesize=auto -O normalization=formD -O relatime=on -O xattr=sa -O devices=off' +export ZFS_POOLS_RAID_TYPE= +export ZFS_NO_INFO_MESSAGES=1 +export ZFS_SWAP_SIZE=2 +export ZFS_FREE_TAIL_SPACE=12 + " + + set -x } trap _exit_hook EXIT } From 8765df0704a83479f811eb6ab465b897e0b303fc Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 25 Apr 2021 22:31:25 +0200 Subject: [PATCH 26/84] Shellcheck cosmetix fixes --- install-zfs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install-zfs.sh b/install-zfs.sh index fe50929..130750c 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -464,7 +464,8 @@ export ZFS_FREE_TAIL_SPACE=$v_free_tail_space" # Convenient ready exports (selecting the first two disks): # - local ready=" + # shellcheck disable=SC2155,SC2012 + local _=" export ZFS_USE_PPA= export ZFS_SELECTED_DISKS=$(ls -l /dev/disk/by-id/ | perl -ane 'print "/dev/disk/by-id/@F[8]," if ! /\d$/ && ($c += 1) <= 2' | head -c -1) export ZFS_BOOT_PARTITION_SIZE=2048M From 388ebda257c6c787cfc9b423f4394071d1ebe961 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Mon, 26 Apr 2021 22:48:35 +0200 Subject: [PATCH 27/84] Update encryption to GCM (as per updated guide) --- install-zfs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-zfs.sh b/install-zfs.sh index 130750c..b4b6cdf 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1039,7 +1039,7 @@ function create_pools { set +x if [[ -n $v_passphrase ]]; then - encryption_options=(-O "encryption=on" -O "keylocation=prompt" -O "keyformat=passphrase") + encryption_options=(-O "encryption=aes-256-gcm" -O "keylocation=prompt" -O "keyformat=passphrase") fi set -x From 89964d9bc2aed8f4cf3f4040c01b20b8bf364cfe Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Mon, 26 Apr 2021 22:57:43 +0200 Subject: [PATCH 28/84] Remove official support for Debian --- README.md | 5 ++--- install-zfs.sh | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a585b74..9f3fc09 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # zfs-installer -ZFS installer is a shell script program that fully prepares ZFS on a system, and allows an effortless installation of several Debian-based operating systems using their standard installer (or debootstrap, or any custom script). +ZFS installer is a shell script program that fully prepares ZFS on a system, and allows an effortless installation of several Ubuntu-based operating systems using their standard installer (or debootstrap, or any custom script). - [Requirements and functionality](#requirements-and-functionality) - [Comparison with Ubuntu built-in installer](#comparison-with-ubuntu-built-in-installer) @@ -22,7 +22,6 @@ The program currently supports: - Ubuntu Desktop 18.04.x/20.04 Live - Ubuntu Server 18.04.x/20.04 Live - Linux Mint 19.x, 20 -- Debian 10.x Live (desktop environment required) - ElementaryOS 5.1 The ZFS version installed is 0.8, which supports native encryption and trimming (among the other improvements over 0.7). The required repositories are automatically added to the destination system. @@ -31,7 +30,7 @@ EFI boot is required (any modern (2011+) system will do); legacy boot is current All the ZFS RAID types are supported, with any arbitrary number of disks. An EFI partition is created on each disk, for redundancy purposes. -It's fairly easy to extend the program to support other Debian-based operating systems (e.g. older/newer Ubuntu's, etc.) - the project is (very) open to feature requests. +It's fairly easy to extend the program to support other Ubuntu-based operating systems - the project is open to feature requests. ## Comparison with Ubuntu built-in installer diff --git a/install-zfs.sh b/install-zfs.sh index b4b6cdf..df8b2e2 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -33,7 +33,7 @@ v_free_tail_space= # integer # Variables set during execution -v_linux_distribution= # Debian, Ubuntu, ... WATCH OUT: not necessarily from `lsb_release` (ie. UbuntuServer) +v_linux_distribution= # Ubuntu, LinuxMint, ... WATCH OUT: not necessarily from `lsb_release` (ie. UbuntuServer) v_use_ppa= # 1=true, false otherwise (applies only to Ubuntu-based). v_temp_volume_device= # /dev/zdN; scope: setup_partitions -> sync_os_temp_installation_dir_to_rpool v_suitable_disks=() # (/dev/by-id/disk_id, ...); scope: find_suitable_disks -> select_disk @@ -64,7 +64,7 @@ c_default_rpool_create_options=( ) c_zfs_mount_dir=/mnt c_installed_os_data_mount_dir=/target -declare -A c_supported_linux_distributions=([Debian]=10 [Ubuntu]="18.04 20.04" [UbuntuServer]="18.04 20.04" [LinuxMint]="19.1 19.2 19.3" [Linuxmint]="20 20.1" [elementary]=5.1) +declare -A c_supported_linux_distributions=([Ubuntu]="18.04 20.04" [UbuntuServer]="18.04 20.04" [LinuxMint]="19.1 19.2 19.3" [Linuxmint]="20 20.1" [elementary]=5.1) c_temporary_volume_size=12 # gigabytes; large enough - Debian, for example, takes ~8 GiB. c_passphrase_named_pipe=$(dirname "$(mktemp)")/zfs-installer.pp.fifo From 6143e1604baa589ee1f424ec4d06427dcb370d60 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Mon, 26 Apr 2021 23:23:46 +0200 Subject: [PATCH 29/84] Include all disks in the disk log --- install-zfs.sh | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index df8b2e2..b7871ed 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -321,6 +321,25 @@ In such cases, the module building may fail abruptly, either without visible err fi } +function save_disks_log { + print_step_info_header + + # shellcheck disable=SC2012 # `ls` may clean the output, but in this case, it doesn't matter + ls -l /dev/disk/by-id | tail -n +2 | perl -lane 'print "@F[8..10]"' > "$c_disks_log" + + all_disk_ids=$(find /dev/disk/by-id -mindepth 1 -regextype awk -not -regex '.+-part[0-9]+$' | sort) + + while read -r disk_id || [[ -n $disk_id ]]; do + cat >> "$c_disks_log" << LOG + +## DEVICE: $disk_id ################################ + +$(udevadm info --query=property "$(readlink -f "$disk_id")") + +LOG + done < <(echo -n "$all_disk_ids") +} + function find_suitable_disks { print_step_info_header @@ -329,9 +348,6 @@ function find_suitable_disks { # udevadm trigger - # shellcheck disable=SC2012 # `ls` may clean the output, but in this case, it doesn't matter - ls -l /dev/disk/by-id | tail -n +2 | perl -lane 'print "@F[8..10]"' > "$c_disks_log" - local candidate_disk_ids local mounted_devices @@ -361,14 +377,6 @@ function find_suitable_disks { v_suitable_disks+=("$disk_id") fi fi - - cat >> "$c_disks_log" << LOG - -## DEVICE: $disk_id ################################ - -$(udevadm info --query=property "$(readlink -f "$disk_id")") - -LOG done < <(echo -n "$candidate_disk_ids") if [[ ${#v_suitable_disks[@]} -eq 0 ]]; then @@ -1490,6 +1498,7 @@ store_running_processes check_prerequisites display_intro_banner check_system_memory +save_disks_log find_suitable_disks distro_dependent_invoke "set_zfs_ppa_requirement" register_exit_hook From a31d94bfceb6fc90b0dd48a1b145b5316853af24 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Tue, 27 Apr 2021 12:40:57 +0200 Subject: [PATCH 30/84] Cosmetic improvement to distro_dependent_invoke() --- install-zfs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install-zfs.sh b/install-zfs.sh index b7871ed..b0e5bbe 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -110,11 +110,12 @@ c_udevadm_settle_timeout=10 # seconds # function distro_dependent_invoke { local distro_specific_fx_name="$1_$v_linux_distribution" + local invoke_option=${2:-} if declare -f "$distro_specific_fx_name" > /dev/null; then "$distro_specific_fx_name" else - if ! declare -f "$1" > /dev/null && [[ "${2:-}" == "--noforce" ]]; then + if ! declare -f "$1" > /dev/null && [[ $invoke_option == "--noforce" ]]; then : # do nothing else "$1" From 076098de3a2f8e1662515c781152ef7e71021dcb Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Tue, 27 Apr 2021 12:48:05 +0200 Subject: [PATCH 31/84] distro_dependent_invoke(): Add option checking --- install-zfs.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/install-zfs.sh b/install-zfs.sh index b0e5bbe..44cabd9 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -112,6 +112,11 @@ function distro_dependent_invoke { local distro_specific_fx_name="$1_$v_linux_distribution" local invoke_option=${2:-} + if [[ ! $invoke_option =~ ^(|--noforce)$ ]]; then + >&2 echo "Invalid distro_dependent_invoke() option: $invoke_option" + exit 1 + fi + if declare -f "$distro_specific_fx_name" > /dev/null; then "$distro_specific_fx_name" else From 4a7fd206f1ffca6ec49c82374e947a456146a390 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 5 Jun 2021 22:25:08 +0200 Subject: [PATCH 32/84] Ubuntu Server: fix the udev* systemd units (re)start Using a glob requires `--all` in order to work as intended, otherwise, it doesn't start units that are not loaded already. --- install-zfs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-zfs.sh b/install-zfs.sh index 44cabd9..3b90b9e 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -826,7 +826,7 @@ function install_host_packages_UbuntuServer { umount /lib/modules rm -r /lib/modules ln -s /tmp/modules /lib - systemctl start 'systemd-udevd*' + systemctl start --all 'systemd-udevd*' # Additionally, the linux packages for the running kernel are not installed, at least when # the standard installation is performed. Didn't test on the HWE option; if it's not required, From 492c94f3eab68886092cc97eb8a173e5ae3f1963 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 5 Jun 2021 23:12:21 +0200 Subject: [PATCH 33/84] Improve message about module compilation requirements The O/S is related up to a certain extent; hardware threads have instead a central role. --- install-zfs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-zfs.sh b/install-zfs.sh index 3b90b9e..e657c72 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -319,7 +319,7 @@ function check_system_memory { # local dialog_message='WARNING! In some cases, the ZFS modules require compilation. -On systems with relatively little RAM, the procedure may crash during the compilation, for example with 3 GB on Debian 10.9. +On systems with relatively little RAM and many hardware threads, the procedure may crash during the compilation (e.g. 3 GB/16 threads). In such cases, the module building may fail abruptly, either without visible errors (leaving "process killed" messages in the syslog), or with package installation errors (leaving odd errors in the module'\''s `make.log`).' From 2488d51f95c062a7d7958bf0763bce450dc3dc87 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 6 Jun 2021 01:39:19 +0200 Subject: [PATCH 34/84] Disable PPA on Ubuntu Server See #200. --- install-zfs.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/install-zfs.sh b/install-zfs.sh index e657c72..46a5170 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -282,6 +282,11 @@ function check_prerequisites { elif [[ ! ${c_supported_linux_distributions["$v_linux_distribution"]} =~ $distro_version_regex ]]; then echo "This Linux distribution version ($v_linux_version) is not supported; supported versions: ${c_supported_linux_distributions["$v_linux_distribution"]}" exit 1 + elif [[ ${ZFS_USE_PPA:-} == "1" && $v_linux_distribution == "UbuntuServer" ]]; then + # As of Jun/2021, it breaks the installation. + # + echo "The PPA is not (currently) supported on Ubuntu Server!" + exit 1 fi set +x From c4cf5181d14e431f6281ff97493b1368fec1354d Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 13 Jun 2021 15:38:59 +0200 Subject: [PATCH 35/84] Enable autotrim feature on both pools, and remove trimming cron job --- README.md | 5 ++--- install-zfs.sh | 42 ++---------------------------------------- 2 files changed, 4 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 9f3fc09..6e29286 100644 --- a/README.md +++ b/README.md @@ -41,11 +41,10 @@ The advantages of this project over the Ubuntu installer are: 1. it supports pools configuration; 2. it allows specifying the RAID type; 3. it allows customization of the disk partitions; -4. it supports additional features (e.g. encryption); +4. it supports additional features (e.g. encryption and trimming); 5. it supports many more operating systems; 6. it supports unattended installations, via custom scripts; -7. it installs a convenient trimming job for ZFS pools; -8. it's easy to extend. +7. it's easy to extend. The disadvantages are: diff --git a/install-zfs.sh b/install-zfs.sh index 46a5170..fbf9051 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -50,10 +50,12 @@ c_default_boot_partition_size=2048 # megabytes c_memory_warning_limit=$((3584 - 128)) # megabytes; exclude some RAM, which can be occupied/shared c_default_bpool_create_options=( -o ashift=12 + -o autotrim=on -O devices=off ) c_default_rpool_create_options=( -o ashift=12 + -o autotrim=on -O acltype=posixacl -O compression=lz4 -O dnodesize=auto @@ -1404,45 +1406,6 @@ function update_zed_cache_Debian { chroot_execute "sed -Ei 's|$c_installed_os_data_mount_dir/?|/|' /etc/zfs/zfs-list.cache/$v_rpool_name" } -# We don't care about synchronizing with the `fstrim` service for two reasons: -# -# - we assume that there are no other (significantly) large filesystems; -# - trimming is fast (takes minutes on a 1 TB disk). -# -# The code is a straight copy of the `fstrim` service. -# -function configure_pools_trimming { - print_step_info_header - - chroot_execute "cat > /lib/systemd/system/zfs-trim.service << UNIT -[Unit] -Description=Discard unused ZFS blocks -ConditionVirtualization=!container - -[Service] -Type=oneshot -ExecStart=/sbin/zpool trim $c_bpool_name -ExecStart=/sbin/zpool trim $v_rpool_name -UNIT" - - chroot_execute " cat > /lib/systemd/system/zfs-trim.timer << TIMER -[Unit] -Description=Discard unused ZFS blocks once a week -ConditionVirtualization=!container - -[Timer] -OnCalendar=weekly -AccuracySec=1h -Persistent=true - -[Install] -WantedBy=timers.target -TIMER" - - chroot_execute "systemctl daemon-reload" - chroot_execute "systemctl enable zfs-trim.timer" -} - function configure_remaining_settings { print_step_info_header @@ -1552,7 +1515,6 @@ sync_efi_partitions configure_boot_pool_import update_initramfs distro_dependent_invoke "update_zed_cache" --noforce -configure_pools_trimming configure_remaining_settings prepare_for_system_exit From caf59dc23f0ce6e66ceee10dece237c5a32b342a Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 13 Jun 2021 15:41:57 +0200 Subject: [PATCH 36/84] Handle Ubiquity now emptying `/run` --- install-zfs.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index fbf9051..94928f2 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1130,13 +1130,17 @@ function sync_os_temp_installation_dir_to_rpool { perl -lane 'BEGIN { $/ = "\r"; $|++ } $F[1] =~ /(\d+)%$/ && print $1' | whiptail --gauge "Syncing the installed O/S to the root pool FS..." 30 100 0 - mkdir "$c_zfs_mount_dir/run" - - # Required destination of symlink `/etc/resolv.conf`, present in Ubuntu systems (not Debian). + # Ubiquity used to leave `/target/run`, which included the file symlinked by `/etc/resolv.conf`. At + # some point, it started cleaning it after installation, leaving the symlink broken, which caused + # the jail preparation to fail. For this reason, we now create the dir/file manually. + # As of Jun/2021, it's not clear if there is any O/S leaving the dir/file, but for simplicity, we + # always create them if not existing. # - if [[ -d $c_installed_os_data_mount_dir/run/systemd/resolve ]]; then - rsync -av --relative "$c_installed_os_data_mount_dir/run/./systemd/resolve" "$c_zfs_mount_dir/run" - fi + mkdir -p "$c_installed_os_data_mount_dir/run/systemd/resolve" + touch "$c_installed_os_data_mount_dir/run/systemd/resolve/stub-resolv.conf" + + mkdir "$c_zfs_mount_dir/run" + rsync -av --relative "$c_installed_os_data_mount_dir/run/./systemd/resolve" "$c_zfs_mount_dir/run" umount "$c_installed_os_data_mount_dir" } From ebd0f38f58f7c62aa4fb6029c29bfcfb9e4221c8 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 13 Jun 2021 16:05:15 +0200 Subject: [PATCH 37/84] Remove global Shellcheck directives That issues have been fixed. --- install-zfs.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/install-zfs.sh b/install-zfs.sh index 94928f2..df49079 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1,5 +1,4 @@ #!/bin/bash -# shellcheck disable=SC2015,SC2016 # Shellcheck issue descriptions: # From fec9ce1e815589ce6bb82fa7334b25b4193e4e7b Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 13 Jun 2021 15:54:16 +0200 Subject: [PATCH 38/84] Officially close project --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 6e29286..f69f732 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ ZFS installer is a shell script program that fully prepares ZFS on a system, and allows an effortless installation of several Ubuntu-based operating systems using their standard installer (or debootstrap, or any custom script). +- [Status](#status) - [Requirements and functionality](#requirements-and-functionality) - [Comparison with Ubuntu built-in installer](#comparison-with-ubuntu-built-in-installer) - [Instructions](#instructions) @@ -15,6 +16,21 @@ ZFS installer is a shell script program that fully prepares ZFS on a system, and - [Help](#help) - [Credits](#credits) +## Status + +**WARNING: THIS PROJECT HAS BEEN PUT ON HOLD, AND IT'S NOT STABLE ANYMORE** + +Working on this type of program is extremely time consuming, for a few reasons: + +1. installers are sometimes updated (even within the same Ubuntu patch version), breaking the program in unpredictable ways; +2. the problem above is compounded by different distros having minor differences in configuration, which again, break the program; +3. it's not possible to automate the testing without a considerably sophisticated tool (which should, in theory, perform the operations on the Ubiquity GUI; this is also impossible for Ubuntu Server); while debootstrap installations can be programmatically tested, the bulk of the bugs are related to the GUI installer +4. I'm the only developer actively working on the project; the ZFS are collaborating with Canonical, and, while the outcome is highly desirable for ZFS, it makes manual execution the only option for even minor customization + +Therefore, I'm closing this project. I may keep developing in order to support my home installation, but I can't give any guarantee. + +I'm open to PR, though. + ## Requirements and functionality The program currently supports: From c336baeb90958076609a88504d1b5acd4ef6366f Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 09:28:40 +0200 Subject: [PATCH 39/84] CI: Move CI code into a single script --- .travis.yml | 8 +------- ci/run_shellcheck.sh | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 7 deletions(-) create mode 100755 ci/run_shellcheck.sh diff --git a/.travis.yml b/.travis.yml index 1194247..27cee52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,10 +5,4 @@ cache: directories: - "/opt/shellcheck" script: -- mkdir -p /opt/shellcheck -- "[[ ! -e /opt/shellcheck/shellcheck ]] && wget -qO- https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz - | tar -xJv -O shellcheck-stable/shellcheck | sudo tee /opt/shellcheck/shellcheck - > /dev/null || true" -- sudo chmod +x /opt/shellcheck/shellcheck -- "/opt/shellcheck/shellcheck --version" -- "/opt/shellcheck/shellcheck $(grep -lzP '^#!/bin/\\w+sh' -r .)" +- ci/run_shellcheck diff --git a/ci/run_shellcheck.sh b/ci/run_shellcheck.sh new file mode 100755 index 0000000..0b32cbc --- /dev/null +++ b/ci/run_shellcheck.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -o pipefail +set -o errexit +set -o nounset +set -o errtrace +shopt -s inherit_errexit + +ci/run_shellcheck mkdir -p /opt/shellcheck +[[ ! -e /opt/shellcheck/shellcheck ]] && wget -qO- https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz + | tar -xJv -O shellcheck-stable/shellcheck | sudo tee /opt/shellcheck/shellcheck + > /dev/null || true +sudo chmod +x /opt/shellcheck/shellcheck +/opt/shellcheck/shellcheck --version +/opt/shellcheck/shellcheck $(grep -lzP '^#!/bin/\\w+sh' -r .) From 26720f38d383215870ad05d970bf649257b9bd44 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 09:30:46 +0200 Subject: [PATCH 40/84] Improvements to CI script --- ci/run_shellcheck.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/ci/run_shellcheck.sh b/ci/run_shellcheck.sh index 0b32cbc..cab0bca 100755 --- a/ci/run_shellcheck.sh +++ b/ci/run_shellcheck.sh @@ -6,10 +6,16 @@ set -o nounset set -o errtrace shopt -s inherit_errexit -ci/run_shellcheck mkdir -p /opt/shellcheck -[[ ! -e /opt/shellcheck/shellcheck ]] && wget -qO- https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz - | tar -xJv -O shellcheck-stable/shellcheck | sudo tee /opt/shellcheck/shellcheck - > /dev/null || true -sudo chmod +x /opt/shellcheck/shellcheck +if [[ ! -e /opt/shellcheck/shellcheck ]]; then + mkdir -p /opt/shellcheck + + wget -qO- https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz \ + | tar xJv -O shellcheck-stable/shellcheck \ + > /opt/shellcheck/shellcheck + + chmod +x /opt/shellcheck/shellcheck +fi + /opt/shellcheck/shellcheck --version -/opt/shellcheck/shellcheck $(grep -lzP '^#!/bin/\\w+sh' -r .) + +grep -lZP '^#!/bin/\w+sh' -R | xargs -0 /opt/shellcheck/shellcheck From 3befeda76e906d7df0252ccce63c4b6c61ed55d5 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 09:56:14 +0200 Subject: [PATCH 41/84] CI: Always download the latest Shellcheck version The previous logic was also semantically incorrect, as the file was never cached, therefore, the `if` was redundant. --- ci/run_shellcheck.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/ci/run_shellcheck.sh b/ci/run_shellcheck.sh index cab0bca..a54d11b 100755 --- a/ci/run_shellcheck.sh +++ b/ci/run_shellcheck.sh @@ -6,15 +6,19 @@ set -o nounset set -o errtrace shopt -s inherit_errexit -if [[ ! -e /opt/shellcheck/shellcheck ]]; then - mkdir -p /opt/shellcheck +# Always download the latest version: +# +# - it's fast and stable enough not to worry about it; +# - the workflow is basically single-person, so there's no risk of a new dev encountering an error found +# by a new shellcheck version. - wget -qO- https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz \ - | tar xJv -O shellcheck-stable/shellcheck \ - > /opt/shellcheck/shellcheck +mkdir -p /opt/shellcheck - chmod +x /opt/shellcheck/shellcheck -fi +wget -qO- https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz \ + | tar xJv -O shellcheck-stable/shellcheck \ + > /opt/shellcheck/shellcheck + +chmod +x /opt/shellcheck/shellcheck /opt/shellcheck/shellcheck --version From e103ff4d3f4ba89da4272465a7b5af8ed2220207 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 13:12:00 +0200 Subject: [PATCH 42/84] README: Remove `sh` code block formatting for options section Gets messy in some cases (e.g. with single quotes), and doesn't make much sense, since there is only one command. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f69f732..5134393 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ The installer itself can run over SSH (\[S\]Ubiquity of course needs to be still The program supports unattended installation, via environment variables. The program built-in help explains all the options: -```sh +``` $ wget -qO- https://git.io/JelI5 | bash /dev/stdin --help Usage: install-zfs.sh [-h|--help] From 0fc633bd54fee3aa26399d4f3eaca6c5d90cbd78 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 13:14:20 +0200 Subject: [PATCH 43/84] README: Update in relation the the "Use PPA" new option --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5134393..86e2e65 100644 --- a/README.md +++ b/README.md @@ -57,10 +57,11 @@ The advantages of this project over the Ubuntu installer are: 1. it supports pools configuration; 2. it allows specifying the RAID type; 3. it allows customization of the disk partitions; -4. it supports additional features (e.g. encryption and trimming); -5. it supports many more operating systems; -6. it supports unattended installations, via custom scripts; -7. it's easy to extend. +4. it supports new OpenZFS versions, via PPA `jonathonf/zfs`. +6. it supports additional features (e.g. encryption and trimming); +7. it supports many more operating systems; +7. it supports unattended installations, via custom scripts; +8. it's easy to extend. The disadvantages are: @@ -110,6 +111,7 @@ This script needs to be run with admin permissions, from a Live CD. The procedure can be entirely automated via environment variables: - ZFS_OS_INSTALLATION_SCRIPT : path of a script to execute instead of Ubiquity (see dedicated section below) +- ZFS_USE_PPA : set to 1 to use packages from `ppa:jonathonf/zfs` (automatically set to true if the O/S version doesn't ship at least v0.8) - ZFS_SELECTED_DISKS : full path of the devices to create the pool on, comma-separated - ZFS_PASSPHRASE - ZFS_RPOOL_NAME From b0c25858499b380c76db99b49bd777bfa514ef61 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 14:39:22 +0200 Subject: [PATCH 44/84] README: Simplify markdown for list of advantages --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 86e2e65..c042eeb 100644 --- a/README.md +++ b/README.md @@ -55,13 +55,13 @@ As of 20.04, Canonical makes available an experimental ZFS installer on Ubuntu D The advantages of this project over the Ubuntu installer are: 1. it supports pools configuration; -2. it allows specifying the RAID type; -3. it allows customization of the disk partitions; -4. it supports new OpenZFS versions, via PPA `jonathonf/zfs`. -6. it supports additional features (e.g. encryption and trimming); -7. it supports many more operating systems; -7. it supports unattended installations, via custom scripts; -8. it's easy to extend. +1. it allows specifying the RAID type; +1. it allows customization of the disk partitions; +1. it supports new OpenZFS versions, via PPA `jonathonf/zfs`. +1. it supports additional features (e.g. encryption and trimming); +1. it supports many more operating systems; +1. it supports unattended installations, via custom scripts; +1. it's easy to extend. The disadvantages are: From e6faa246c29b8811dd52e8ff4759f3d2d8dd191c Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 25 Apr 2021 21:01:35 +0200 Subject: [PATCH 45/84] Cosmetic: Simplify GRUB editing expressions --- install-zfs.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index df49079..ee09467 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1285,9 +1285,9 @@ function configure_and_update_grub { # chroot_execute "perl -i -pe 's/(GRUB_TIMEOUT_STYLE=hidden)/#\$1/' /etc/default/grub" chroot_execute "perl -i -pe 's/^(GRUB_HIDDEN_.*)/#\$1/' /etc/default/grub" - chroot_execute "perl -i -pe 's/(GRUB_TIMEOUT=)0/\${1}5/' /etc/default/grub" - chroot_execute "perl -i -pe 's/(GRUB_CMDLINE_LINUX_DEFAULT=.*)quiet/\$1/' /etc/default/grub" - chroot_execute "perl -i -pe 's/(GRUB_CMDLINE_LINUX_DEFAULT=.*)splash/\$1/' /etc/default/grub" + chroot_execute "perl -i -pe 's/GRUB_TIMEOUT=\K0/5/' /etc/default/grub" + chroot_execute "perl -i -pe 's/GRUB_CMDLINE_LINUX_DEFAULT=.*\Kquiet//' /etc/default/grub" + chroot_execute "perl -i -pe 's/GRUB_CMDLINE_LINUX_DEFAULT=.*\Ksplash//' /etc/default/grub" chroot_execute "perl -i -pe 's/#(GRUB_TERMINAL=console)/\$1/' /etc/default/grub" chroot_execute 'echo "GRUB_RECORDFAIL_TIMEOUT=5" >> /etc/default/grub' @@ -1304,7 +1304,7 @@ function configure_and_update_grub_Debian { chroot_execute "perl -i -pe 's/GRUB_CMDLINE_LINUX_DEFAULT=\"\K/init_on_alloc=0 /' /etc/default/grub" chroot_execute "perl -i -pe 's/(GRUB_CMDLINE_LINUX=\")/\${1}root=ZFS=$v_rpool_name /' /etc/default/grub" - chroot_execute "perl -i -pe 's/(GRUB_CMDLINE_LINUX_DEFAULT=.*)quiet/\$1/' /etc/default/grub" + chroot_execute "perl -i -pe 's/GRUB_CMDLINE_LINUX_DEFAULT=.*\Kquiet//' /etc/default/grub" chroot_execute "perl -i -pe 's/#(GRUB_TERMINAL=console)/\$1/' /etc/default/grub" chroot_execute "update-grub" From 353009a700623835ed51d5e3143c5ee0f17e830b Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 25 Apr 2021 21:36:46 +0200 Subject: [PATCH 46/84] Add a few missing print_step_info_header() invocations --- install-zfs.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/install-zfs.sh b/install-zfs.sh index ee09467..9fe0ce7 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -454,10 +454,14 @@ function set_zfs_ppa_requirement_Linuxmint { # from the logs. # function create_passphrase_named_pipe { + print_step_info_header + mkfifo "$c_passphrase_named_pipe" } function register_exit_hook { + print_step_info_header + function _exit_hook { rm -f "$c_passphrase_named_pipe" @@ -1096,6 +1100,8 @@ function create_pools { } function create_swap_volume { + print_step_info_header + if [[ $v_swap_size -gt 0 ]]; then zfs create \ -V "${v_swap_size}G" -b "$(getconf PAGESIZE)" \ From 04e634189da44abf268bc1d973589f9f66375797 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 25 Apr 2021 22:21:09 +0200 Subject: [PATCH 47/84] Debian: Handle preexisting zed symlink In the latest test (on 10.9 w/ ZFS 2.0.3), the file already existed. --- install-zfs.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/install-zfs.sh b/install-zfs.sh index 9fe0ce7..5521062 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1375,7 +1375,10 @@ function update_initramfs { function update_zed_cache_Debian { chroot_execute "mkdir /etc/zfs/zfs-list.cache" chroot_execute "touch /etc/zfs/zfs-list.cache/$v_rpool_name" - chroot_execute "ln -s /usr/lib/zfs-linux/zed.d/history_event-zfs-list-cacher.sh /etc/zfs/zed.d/" + + # On Debian, this file may exist already. + # + chroot_execute "[[ ! -f /etc/zfs/zed.d/history_event-zfs-list-cacher.sh ]] && ln -s /usr/lib/zfs-linux/zed.d/history_event-zfs-list-cacher.sh /etc/zfs/zed.d/" # Assumed to be present by the zedlet above, but missing. # Filed issue: https://github.com/zfsonlinux/zfs/issues/9945. From 13ee0fd704b1fbd208ca7b90838950130b722732 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Mon, 26 Apr 2021 22:20:55 +0200 Subject: [PATCH 48/84] Merge Debian-specific GRUB configuration with regular one There isn't any significant difference. NOTE: The Debian installation may be currently broken. --- install-zfs.sh | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index 5521062..618aeef 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1304,18 +1304,6 @@ function configure_and_update_grub { chroot_execute "update-grub" } -function configure_and_update_grub_Debian { - print_step_info_header - - chroot_execute "perl -i -pe 's/GRUB_CMDLINE_LINUX_DEFAULT=\"\K/init_on_alloc=0 /' /etc/default/grub" - - chroot_execute "perl -i -pe 's/(GRUB_CMDLINE_LINUX=\")/\${1}root=ZFS=$v_rpool_name /' /etc/default/grub" - chroot_execute "perl -i -pe 's/GRUB_CMDLINE_LINUX_DEFAULT=.*\Kquiet//' /etc/default/grub" - chroot_execute "perl -i -pe 's/#(GRUB_TERMINAL=console)/\$1/' /etc/default/grub" - - chroot_execute "update-grub" -} - function sync_efi_partitions { print_step_info_header @@ -1522,7 +1510,7 @@ fi prepare_jail distro_dependent_invoke "install_jail_zfs_packages" prepare_efi_partition -distro_dependent_invoke "configure_and_update_grub" +configure_and_update_grub sync_efi_partitions configure_boot_pool_import update_initramfs From b9cd79a02f8d351061e1c25303317e2dc18a52d8 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Tue, 27 Apr 2021 13:10:03 +0200 Subject: [PATCH 49/84] New function invocation mechanism (README!) The new mechanism is clearly explained in the comment. The rationale is the next implementation of hot swapping. WATCH OUT! The semantics have changed. Previously, it was evident by looking at the invocation when a function had distro-based overrides; now, that needs to be looked into. Since this behavior is not very impactful, it's not been ported, for simplicity purposes. --- install-zfs.sh | 130 +++++++++++++++++++++++++------------------------ 1 file changed, 67 insertions(+), 63 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index 618aeef..5ce2ce3 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -89,43 +89,47 @@ c_udevadm_settle_timeout=10 # seconds # HELPER FUNCTIONS ############################################################# -# Chooses a function and invokes it depending on the O/S distribution. +# Invoke a function, with a primitive dynamic dispatch based on the distribution. # -# Example: +# Format: `invoke "function" [--optional]`. +# +# A target function must exist, otherwise a error is raised, unless `--optional` is specified. +# `--optional` is useful when a step is specific to a single distribution, e.g. Debian's root password. +# +# Examples: # # $ function install_jail_zfs_packages { :; } # $ function install_jail_zfs_packages_Debian { :; } # $ distro_dependent_invoke "install_jail_zfs_packages" # -# If the distribution is `Debian`, the second will be invoked, otherwise, the -# first. -# -# If the function is invoked with `--noforce` as second parameter, and there is -# no matching function: +# If the distribution is `Debian`, the second will be invoked, otherwise, the first. # # $ function update_zed_cache_Ubuntu { :; } -# $ distro_dependent_invoke "install_jail_zfs_packages" --noforce +# $ distro_dependent_invoke "update_zed_cache" --optional # -# then nothing happens. Without `--noforce`, this invocation will cause an -# error. +# If the distribution is `Debian`, nothing will happen. # -function distro_dependent_invoke { - local distro_specific_fx_name="$1_$v_linux_distribution" +# $ function update_zed_cache_Ubuntu { :; } +# $ distro_dependent_invoke "update_zed_cache" +# +# If the distribution is `Debian`, an error will be raised. +# +function invoke { + local base_fx_name=$1 + local distro_specific_fx_name=$1_$v_linux_distribution local invoke_option=${2:-} - if [[ ! $invoke_option =~ ^(|--noforce)$ ]]; then - >&2 echo "Invalid distro_dependent_invoke() option: $invoke_option" + if [[ ! $invoke_option =~ ^(|--optional)$ ]]; then + >&2 echo "Invalid invoke() option: $invoke_option" exit 1 fi + # Invoke it regardless when it's not optional. + if declare -f "$distro_specific_fx_name" > /dev/null; then "$distro_specific_fx_name" - else - if ! declare -f "$1" > /dev/null && [[ $invoke_option == "--noforce" ]]; then - : # do nothing - else - "$1" - fi + elif declare -f "$base_fx_name" > /dev/null || [[ ! $invoke_option == "--optional" ]]; then + "$base_fx_name" fi } @@ -1465,57 +1469,57 @@ if [[ $# -ne 0 ]]; then display_help_and_exit fi -activate_debug -set_distribution_data -distro_dependent_invoke "store_os_distro_information" -store_running_processes -check_prerequisites -display_intro_banner -check_system_memory -save_disks_log -find_suitable_disks -distro_dependent_invoke "set_zfs_ppa_requirement" -register_exit_hook -create_passphrase_named_pipe +invoke "activate_debug" +invoke "set_distribution_data" +invoke "store_os_distro_information" +invoke "store_running_processes" +invoke "check_prerequisites" +invoke "display_intro_banner" +invoke "check_system_memory" +invoke "save_disks_log" +invoke "find_suitable_disks" +invoke "set_zfs_ppa_requirement" +invoke "register_exit_hook" +invoke "create_passphrase_named_pipe" -select_disks -select_pools_raid_type -distro_dependent_invoke "ask_root_password" --noforce -ask_encryption -ask_boot_partition_size -ask_swap_size -ask_free_tail_space -ask_rpool_name -ask_pool_create_options +invoke "select_disks" +invoke "select_pools_raid_type" +invoke "ask_root_password" --optional +invoke "ask_encryption" +invoke "ask_boot_partition_size" +invoke "ask_swap_size" +invoke "ask_free_tail_space" +invoke "ask_rpool_name" +invoke "ask_pool_create_options" -distro_dependent_invoke "install_host_packages" -setup_partitions +invoke "install_host_packages" +invoke "setup_partitions" if [[ "${ZFS_OS_INSTALLATION_SCRIPT:-}" == "" ]]; then # Includes the O/S extra configuration, if necessary (network, root pwd, etc.) - distro_dependent_invoke "install_operating_system" + invoke "install_operating_system" - create_pools - create_swap_volume - sync_os_temp_installation_dir_to_rpool - remove_temp_partition_and_expand_rpool + invoke "create_pools" + invoke "create_swap_volume" + invoke "sync_os_temp_installation_dir_to_rpool" + invoke "remove_temp_partition_and_expand_rpool" else - create_pools - create_swap_volume - remove_temp_partition_and_expand_rpool + invoke "create_pools" + invoke "create_swap_volume" + invoke "remove_temp_partition_and_expand_rpool" - custom_install_operating_system + invoke "custom_install_operating_system" fi -prepare_jail -distro_dependent_invoke "install_jail_zfs_packages" -prepare_efi_partition -configure_and_update_grub -sync_efi_partitions -configure_boot_pool_import -update_initramfs -distro_dependent_invoke "update_zed_cache" --noforce -configure_remaining_settings +invoke "prepare_jail" +invoke "install_jail_zfs_packages" +invoke "prepare_efi_partition" +invoke "configure_and_update_grub" +invoke "sync_efi_partitions" +invoke "configure_boot_pool_import" +invoke "update_initramfs" +invoke "update_zed_cache" --optional +invoke "configure_remaining_settings" -prepare_for_system_exit -display_exit_banner +invoke "prepare_for_system_exit" +invoke "display_exit_banner" From 02d1e277d24bcf0ca00a3efa87da3f64148305aa Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Tue, 27 Apr 2021 13:17:18 +0200 Subject: [PATCH 50/84] Refactoring: The info header function is now invoked by invoke() --- install-zfs.sh | 103 ++++--------------------------------------------- 1 file changed, 7 insertions(+), 96 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index 5ce2ce3..0670fd9 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -127,21 +127,22 @@ function invoke { # Invoke it regardless when it's not optional. if declare -f "$distro_specific_fx_name" > /dev/null; then + print_step_info_header "$distro_specific_fx_name" + "$distro_specific_fx_name" elif declare -f "$base_fx_name" > /dev/null || [[ ! $invoke_option == "--optional" ]]; then + print_step_info_header "$base_fx_name" + "$base_fx_name" fi } -# shellcheck disable=SC2120 # allow parameters passing even if no calls pass any function print_step_info_header { + local function_name=$1 + echo -n " ############################################################################### -# ${FUNCNAME[1]}" - - [[ "${1:-}" != "" ]] && echo -n " $1" || true - - echo " +# $function_name ############################################################################### " } @@ -223,8 +224,6 @@ Root pool default create options: '"${c_default_rpool_create_options[*]/#-/$'\n' } function activate_debug { - print_step_info_header - mkdir -p "$c_log_dir" exec 5> "$c_install_log" @@ -243,8 +242,6 @@ function set_distribution_data { } function store_os_distro_information { - print_step_info_header - lsb_release --all > "$c_os_information_log" # Madness, in order not to force the user to invoke "sudo -E". @@ -268,8 +265,6 @@ function store_running_processes { } function check_prerequisites { - print_step_info_header - local distro_version_regex=\\b${v_linux_version//./\\.}\\b if [[ ! -d /sys/firmware/efi ]]; then @@ -305,8 +300,6 @@ function check_prerequisites { } function display_intro_banner { - print_step_info_header - local dialog_message='Hello! This script will prepare the ZFS pools on the system, install Ubuntu, and configure the boot. @@ -338,8 +331,6 @@ In such cases, the module building may fail abruptly, either without visible err } function save_disks_log { - print_step_info_header - # shellcheck disable=SC2012 # `ls` may clean the output, but in this case, it doesn't matter ls -l /dev/disk/by-id | tail -n +2 | perl -lane 'print "@F[8..10]"' > "$c_disks_log" @@ -357,8 +348,6 @@ LOG } function find_suitable_disks { - print_step_info_header - # In some freaky cases, `/dev/disk/by-id` is not up to date, so we refresh. One case is after # starting a VirtualBox VM that is a full clone of a suspended VM with snapshots. # @@ -423,8 +412,6 @@ If you think this is a bug, please open an issue on https://github.com/saveriomi # install_host_packages() and install_host_packages_UbuntuServer(). # function set_zfs_ppa_requirement { - print_step_info_header - apt update local zfs_package_version @@ -447,8 +434,6 @@ function set_zfs_ppa_requirement_Debian { # to it being incorrectly setup). # function set_zfs_ppa_requirement_Linuxmint { - print_step_info_header - perl -i -pe 's/^(deb cdrom)/# $1/' /etc/apt/sources.list set_zfs_ppa_requirement @@ -458,14 +443,10 @@ function set_zfs_ppa_requirement_Linuxmint { # from the logs. # function create_passphrase_named_pipe { - print_step_info_header - mkfifo "$c_passphrase_named_pipe" } function register_exit_hook { - print_step_info_header - function _exit_hook { rm -f "$c_passphrase_named_pipe" @@ -514,8 +495,6 @@ export ZFS_FREE_TAIL_SPACE=12 } function select_disks { - print_step_info_header - if [[ "${ZFS_SELECTED_DISKS:-}" != "" ]]; then mapfile -d, -t v_selected_disks < <(echo -n "$ZFS_SELECTED_DISKS") else @@ -555,8 +534,6 @@ Devices with mounted partitions, cdroms, and removable devices are not displayed } function select_pools_raid_type { - print_step_info_header - local raw_pools_raid_type= if [[ -v ZFS_POOLS_RAID_TYPE ]]; then @@ -600,8 +577,6 @@ function select_pools_raid_type { } function ask_root_password_Debian { - print_step_info_header - set +x if [[ ${ZFS_DEBIAN_ROOT_PASSWORD:-} != "" ]]; then v_root_password="$ZFS_DEBIAN_ROOT_PASSWORD" @@ -620,8 +595,6 @@ function ask_root_password_Debian { } function ask_encryption { - print_step_info_header - set +x if [[ -v ZFS_PASSPHRASE ]]; then @@ -652,8 +625,6 @@ Leave blank to keep encryption disabled. } function ask_boot_partition_size { - print_step_info_header - if [[ ${ZFS_BOOT_PARTITION_SIZE:-} != "" ]]; then v_boot_partition_size=$ZFS_BOOT_PARTITION_SIZE else @@ -672,8 +643,6 @@ Supported formats: '512M', '3G'" 30 100 ${c_default_boot_partition_size}M 3>&1 1 } function ask_swap_size { - print_step_info_header - if [[ ${ZFS_SWAP_SIZE:-} != "" ]]; then v_swap_size=$ZFS_SWAP_SIZE else @@ -690,8 +659,6 @@ function ask_swap_size { } function ask_free_tail_space { - print_step_info_header - if [[ ${ZFS_FREE_TAIL_SPACE:-} != "" ]]; then v_free_tail_space=$ZFS_FREE_TAIL_SPACE else @@ -716,8 +683,6 @@ For detailed informations, see the wiki page: https://github.com/saveriomiroddi/ } function ask_rpool_name { - print_step_info_header - if [[ ${ZFS_RPOOL_NAME:-} != "" ]]; then v_rpool_name=$ZFS_RPOOL_NAME else @@ -734,8 +699,6 @@ function ask_rpool_name { } function ask_pool_create_options { - print_step_info_header - local bpool_create_options_message='Insert the create options for the boot pool The mount-related options are automatically added, and must not be specified.' @@ -756,8 +719,6 @@ The encryption/mount-related options are automatically added, and must not be sp } function install_host_packages { - print_step_info_header - if [[ $v_use_ppa == "1" ]]; then if [[ ${ZFS_SKIP_LIVE_ZFS_MODULE_INSTALL:-} != "1" ]]; then add-apt-repository --yes "$c_ppa" @@ -782,8 +743,6 @@ function install_host_packages { } function install_host_packages_Debian { - print_step_info_header - if [[ ${ZFS_SKIP_LIVE_ZFS_MODULE_INSTALL:-} != "1" ]]; then echo "zfs-dkms zfs-dkms/note-incompatible-licenses note true" | debconf-set-selections @@ -804,16 +763,12 @@ function install_host_packages_Debian { # Differently from Ubuntu, Mint doesn't have the package installed in the live version. # function install_host_packages_Linuxmint { - print_step_info_header - apt install --yes zfsutils-linux install_host_packages } function install_host_packages_elementary { - print_step_info_header - if [[ ${ZFS_SKIP_LIVE_ZFS_MODULE_INSTALL:-} != "1" ]]; then apt update apt install --yes software-properties-common @@ -823,8 +778,6 @@ function install_host_packages_elementary { } function install_host_packages_UbuntuServer { - print_step_info_header - if [[ $v_use_ppa != "1" ]]; then apt install --yes zfsutils-linux efibootmgr @@ -856,8 +809,6 @@ function install_host_packages_UbuntuServer { } function setup_partitions { - print_step_info_header - local required_tail_space=$((v_free_tail_space > c_temporary_volume_size ? v_free_tail_space : c_temporary_volume_size)) for selected_disk in "${v_selected_disks[@]}"; do @@ -913,8 +864,6 @@ function setup_partitions { } function install_operating_system { - print_step_info_header - local dialog_message='The Ubuntu GUI installer will now be launched. Proceed with the configuration as usual, then, at the partitioning stage: @@ -956,8 +905,6 @@ Proceed with the configuration as usual, then, at the partitioning stage: } function install_operating_system_Debian { - print_step_info_header - # The temporary volume size displayed is an approximation of the format used by the installer, # but it's acceptable - the complexity required is not worth (eg. converting hypothetical units, # etc.). @@ -1010,8 +957,6 @@ CONF } function install_operating_system_UbuntuServer { - print_step_info_header - # O/S Installation # # Subiquity is designed to prevent the user from opening a terminal, which is (to say the least) @@ -1053,8 +998,6 @@ You can switch anytime to this terminal, and back, in order to read the instruct } function custom_install_operating_system { - print_step_info_header - sudo "$ZFS_OS_INSTALLATION_SCRIPT" } @@ -1104,8 +1047,6 @@ function create_pools { } function create_swap_volume { - print_step_info_header - if [[ $v_swap_size -gt 0 ]]; then zfs create \ -V "${v_swap_size}G" -b "$(getconf PAGESIZE)" \ @@ -1117,8 +1058,6 @@ function create_swap_volume { } function sync_os_temp_installation_dir_to_rpool { - print_step_info_header - # On Ubuntu Server, `/boot/efi` and `/cdrom` (!!!) are mounted, but they're not needed. # local mount_dir_submounts @@ -1155,8 +1094,6 @@ function sync_os_temp_installation_dir_to_rpool { } function remove_temp_partition_and_expand_rpool { - print_step_info_header - if (( v_free_tail_space < c_temporary_volume_size )); then if [[ $v_free_tail_space -eq 0 ]]; then local resize_reference=100% @@ -1191,8 +1128,6 @@ function remove_temp_partition_and_expand_rpool { } function prepare_jail { - print_step_info_header - for virtual_fs_dir in proc sys dev; do mount --rbind "/$virtual_fs_dir" "$c_zfs_mount_dir/$virtual_fs_dir" done @@ -1203,8 +1138,6 @@ function prepare_jail { # See install_host_packages() for some comments. # function install_jail_zfs_packages { - print_step_info_header - if [[ $v_use_ppa == "1" ]]; then chroot_execute "add-apt-repository --yes $c_ppa" @@ -1229,8 +1162,6 @@ function install_jail_zfs_packages { } function install_jail_zfs_packages_Debian { - print_step_info_header - chroot_execute 'echo "deb http://deb.debian.org/debian buster main contrib" >> /etc/apt/sources.list' chroot_execute 'echo "deb-src http://deb.debian.org/debian buster main contrib" >> /etc/apt/sources.list' @@ -1250,16 +1181,12 @@ APT' } function install_jail_zfs_packages_elementary { - print_step_info_header - chroot_execute "apt install --yes software-properties-common" install_jail_zfs_packages } function install_jail_zfs_packages_UbuntuServer { - print_step_info_header - if [[ $v_use_ppa != "1" ]]; then chroot_execute "apt install --yes zfsutils-linux zfs-initramfs grub-efi-amd64-signed shim-signed" else @@ -1268,8 +1195,6 @@ function install_jail_zfs_packages_UbuntuServer { } function prepare_efi_partition { - print_step_info_header - chroot_execute "echo PARTUUID=$(blkid -s PARTUUID -o value "${v_selected_disks[0]}-part1") /boot/efi vfat nofail,x-systemd.device-timeout=1 0 1 > /etc/fstab" chroot_execute "mkdir -p /boot/efi" @@ -1279,8 +1204,6 @@ function prepare_efi_partition { } function configure_and_update_grub { - print_step_info_header - chroot_execute "perl -i -pe 's/GRUB_CMDLINE_LINUX_DEFAULT=\"\K/init_on_alloc=0 /' /etc/default/grub" chroot_execute "perl -i -pe 's/(GRUB_CMDLINE_LINUX=\")/\${1}root=ZFS=$v_rpool_name /' /etc/default/grub" @@ -1309,8 +1232,6 @@ function configure_and_update_grub { } function sync_efi_partitions { - print_step_info_header - for ((i = 1; i < ${#v_selected_disks[@]}; i++)); do local synced_efi_partition_path="/boot/efi$((i + 1))" @@ -1330,8 +1251,6 @@ function sync_efi_partitions { } function configure_boot_pool_import { - print_step_info_header - chroot_execute "cat > /etc/systemd/system/zfs-import-$c_bpool_name.service <> /etc/fstab" || true chroot_execute "echo RESUME=none > /etc/initramfs-tools/conf.d/resume" } function prepare_for_system_exit { - print_step_info_header - for virtual_fs_dir in dev sys proc; do umount --recursive --force --lazy "$c_zfs_mount_dir/$virtual_fs_dir" done @@ -1452,8 +1365,6 @@ function prepare_for_system_exit { } function display_exit_banner { - print_step_info_header - local dialog_message="The system has been successfully prepared and installed. You now need to perform a hard reset, then enjoy your ZFS system :-)" From 6c1c1b8512f0971a68777c47a7cc5601b2700bb9 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Tue, 27 Apr 2021 13:22:54 +0200 Subject: [PATCH 51/84] Add script hot swapping functionality --- install-zfs.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/install-zfs.sh b/install-zfs.sh index 0670fd9..214a437 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -42,6 +42,8 @@ v_suitable_disks=() # (/dev/by-id/disk_id, ...); scope: find_suitable_d # Note that Linux Mint is "Linuxmint" from v20 onwards. This actually helps, since some operations are # specific to it. +c_hotswap_file=$PWD/install-zfs.hotswap.sh # see hotswap() for an explanation. + c_bpool_name=bpool c_ppa=ppa:jonathonf/zfs c_efi_system_partition_size=512 # megabytes @@ -124,6 +126,8 @@ function invoke { exit 1 fi + hot_swap_script + # Invoke it regardless when it's not optional. if declare -f "$distro_specific_fx_name" > /dev/null; then @@ -137,6 +141,18 @@ function invoke { fi } +# Tee-hee-hee!! +# +# This is extremely useful for debugging long procedures. Since bash scripts can't be modified while +# running, this allows the dev to create a snapshot, and if the script fails after that, resume and +# add the hotswap script, so that the new code will be loaded automatically. +# +function hot_swap_script { + if [[ -f $c_hotswap_file ]]; then + source "$c_hotswap_file" + fi +} + function print_step_info_header { local function_name=$1 From 20f4ea68319cf8e456c89897eb15ecfaec8cc29d Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Tue, 27 Apr 2021 13:43:16 +0200 Subject: [PATCH 52/84] Make the distro-specific functions use invoke(), when calling the parent --- install-zfs.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index 214a437..80408b5 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -452,7 +452,7 @@ function set_zfs_ppa_requirement_Debian { function set_zfs_ppa_requirement_Linuxmint { perl -i -pe 's/^(deb cdrom)/# $1/' /etc/apt/sources.list - set_zfs_ppa_requirement + invoke "set_zfs_ppa_requirement" } # By using a FIFO, we avoid having to hide statements like `echo $v_passphrase | zpoool create ...` @@ -781,7 +781,7 @@ function install_host_packages_Debian { function install_host_packages_Linuxmint { apt install --yes zfsutils-linux - install_host_packages + invoke "install_host_packages" } function install_host_packages_elementary { @@ -790,7 +790,7 @@ function install_host_packages_elementary { apt install --yes software-properties-common fi - install_host_packages + invoke "install_host_packages" } function install_host_packages_UbuntuServer { @@ -1199,14 +1199,14 @@ APT' function install_jail_zfs_packages_elementary { chroot_execute "apt install --yes software-properties-common" - install_jail_zfs_packages + invoke "install_jail_zfs_packages" } function install_jail_zfs_packages_UbuntuServer { if [[ $v_use_ppa != "1" ]]; then chroot_execute "apt install --yes zfsutils-linux zfs-initramfs grub-efi-amd64-signed shim-signed" else - install_jail_zfs_packages + invoke "install_jail_zfs_packages" fi } From 802d49435863bb53feedfde1d6f68097cbc7b37a Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Tue, 27 Apr 2021 19:56:56 +0200 Subject: [PATCH 53/84] README: Mention that ZFS 2.x can be installed --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c042eeb..b036e60 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ The program currently supports: - Linux Mint 19.x, 20 - ElementaryOS 5.1 -The ZFS version installed is 0.8, which supports native encryption and trimming (among the other improvements over 0.7). The required repositories are automatically added to the destination system. +The ZFS version installed is 0.8 (optionally, 2.x), which supports native encryption and trimming (among the other improvements over 0.7). The required repositories are automatically added to the destination system. EFI boot is required (any modern (2011+) system will do); legacy boot is currently not supported. From afccef0e40805326729c6f7a004c99981e13f5ab Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Tue, 27 Apr 2021 18:29:10 +0200 Subject: [PATCH 54/84] Fix Shellcheck complaint about not following the hotswap script --- install-zfs.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install-zfs.sh b/install-zfs.sh index 80408b5..8111d5a 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -149,6 +149,7 @@ function invoke { # function hot_swap_script { if [[ -f $c_hotswap_file ]]; then + # shellcheck disable=1090 # can't follow; the file might not exist anyway. source "$c_hotswap_file" fi } From ef3174a1fbf7d15a5aca4c0ad2baabb8e530ae6c Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Tue, 27 Apr 2021 11:24:26 +0200 Subject: [PATCH 55/84] Cosmetic: Rename `c_installed_os_data_mount_dir` This is actually to make things a bit easier with the datasets definition. --- install-zfs.sh | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index 8111d5a..bde6fef 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -66,7 +66,7 @@ c_default_rpool_create_options=( -O devices=off ) c_zfs_mount_dir=/mnt -c_installed_os_data_mount_dir=/target +c_installed_os_mount_dir=/target declare -A c_supported_linux_distributions=([Ubuntu]="18.04 20.04" [UbuntuServer]="18.04 20.04" [LinuxMint]="19.1 19.2 19.3" [Linuxmint]="20 20.1" [elementary]=5.1) c_temporary_volume_size=12 # gigabytes; large enough - Debian, for example, takes ~8 GiB. c_passphrase_named_pipe=$(dirname "$(mktemp)")/zfs-installer.pp.fifo @@ -914,11 +914,11 @@ Proceed with the configuration as usual, then, at the partitioning stage: # # Note that we assume that the user created only one partition on the temp volume, as expected. # - if ! mountpoint -q "$c_installed_os_data_mount_dir"; then - mount "$v_temp_volume_device" "$c_installed_os_data_mount_dir" + if ! mountpoint -q "$c_installed_os_mount_dir"; then + mount "$v_temp_volume_device" "$c_installed_os_mount_dir" fi - rm -f "$c_installed_os_data_mount_dir/swapfile" + rm -f "$c_installed_os_mount_dir/swapfile" } function install_operating_system_Debian { @@ -949,24 +949,24 @@ Proceed with the configuration as usual, then, at the partitioning stage: DISPLAY=:0 calamares - mkdir -p "$c_installed_os_data_mount_dir" + mkdir -p "$c_installed_os_mount_dir" # Note how in Debian, for reasons currenly unclear, the mount fails if the partition is passed; # it requires the device to be passed. # - mount "${v_temp_volume_device}" "$c_installed_os_data_mount_dir" + mount "${v_temp_volume_device}" "$c_installed_os_mount_dir" # We don't use chroot()_execute here, as it works on $c_zfs_mount_dir (which is synced on a # later stage). # set +x - chroot "$c_installed_os_data_mount_dir" bash -c "echo root:$(printf "%q" "$v_root_password") | chpasswd" + chroot "$c_installed_os_mount_dir" bash -c "echo root:$(printf "%q" "$v_root_password") | chpasswd" set -x # The installer doesn't set the network interfaces, so, for convenience, we do it. # for interface in $(ip addr show | perl -lne '/^\d+: (?!lo:)(\w+)/ && print $1' ); do - cat > "$c_installed_os_data_mount_dir/etc/network/interfaces.d/$interface" < "$c_installed_os_mount_dir/etc/network/interfaces.d/$interface" < Date: Tue, 27 Apr 2021 18:48:31 +0200 Subject: [PATCH 56/84] Cosmetic: Remove unnecessary quotes, and make use of `-z`/`-n` in conditionals --- install-zfs.sh | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index bde6fef..d67bd0d 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -290,7 +290,7 @@ function check_prerequisites { elif [[ $(id -u) -ne 0 ]]; then echo 'This script must be run with administrative privileges!' exit 1 - elif [[ "${ZFS_OS_INSTALLATION_SCRIPT:-}" != "" && ! -x "$ZFS_OS_INSTALLATION_SCRIPT" ]]; then + elif [[ -n ${ZFS_OS_INSTALLATION_SCRIPT:-} && ! -x $ZFS_OS_INSTALLATION_SCRIPT ]]; then echo "The custom O/S installation script provided doesn't exist or is not executable!" exit 1 elif [[ ! -v c_supported_linux_distributions["$v_linux_distribution"] ]]; then @@ -324,7 +324,7 @@ This script will prepare the ZFS pools on the system, install Ubuntu, and config In order to stop the procedure, hit Esc twice during dialogs (excluding yes/no ones), or Ctrl+C while any operation is running. ' - if [[ ${ZFS_NO_INFO_MESSAGES:-} == "" ]]; then + if [[ -z ${ZFS_NO_INFO_MESSAGES:-} ]]; then whiptail --msgbox "$dialog_message" 30 100 fi } @@ -512,7 +512,7 @@ export ZFS_FREE_TAIL_SPACE=12 } function select_disks { - if [[ "${ZFS_SELECTED_DISKS:-}" != "" ]]; then + if [[ -n ${ZFS_SELECTED_DISKS:-} ]]; then mapfile -d, -t v_selected_disks < <(echo -n "$ZFS_SELECTED_DISKS") else while true; do @@ -595,13 +595,13 @@ function select_pools_raid_type { function ask_root_password_Debian { set +x - if [[ ${ZFS_DEBIAN_ROOT_PASSWORD:-} != "" ]]; then + if [[ -n ${ZFS_DEBIAN_ROOT_PASSWORD:-} ]]; then v_root_password="$ZFS_DEBIAN_ROOT_PASSWORD" else local password_invalid_message= local password_repeat=- - while [[ "$v_root_password" != "$password_repeat" || "$v_root_password" == "" ]]; do + while [[ $v_root_password != "$password_repeat" || -z $v_root_password ]]; do v_root_password=$(whiptail --passwordbox "${password_invalid_message}Please enter the root account password (can't be empty):" 30 100 3>&1 1>&2 2>&3) password_repeat=$(whiptail --passwordbox "Please repeat the password:" 30 100 3>&1 1>&2 2>&3) @@ -642,7 +642,7 @@ Leave blank to keep encryption disabled. } function ask_boot_partition_size { - if [[ ${ZFS_BOOT_PARTITION_SIZE:-} != "" ]]; then + if [[ -n ${ZFS_BOOT_PARTITION_SIZE:-} ]]; then v_boot_partition_size=$ZFS_BOOT_PARTITION_SIZE else local boot_partition_size_invalid_message= @@ -660,7 +660,7 @@ Supported formats: '512M', '3G'" 30 100 ${c_default_boot_partition_size}M 3>&1 1 } function ask_swap_size { - if [[ ${ZFS_SWAP_SIZE:-} != "" ]]; then + if [[ -n ${ZFS_SWAP_SIZE:-} ]]; then v_swap_size=$ZFS_SWAP_SIZE else local swap_size_invalid_message= @@ -676,7 +676,7 @@ function ask_swap_size { } function ask_free_tail_space { - if [[ ${ZFS_FREE_TAIL_SPACE:-} != "" ]]; then + if [[ -n ${ZFS_FREE_TAIL_SPACE:-} ]]; then v_free_tail_space=$ZFS_FREE_TAIL_SPACE else local tail_space_invalid_message= @@ -700,7 +700,7 @@ For detailed informations, see the wiki page: https://github.com/saveriomiroddi/ } function ask_rpool_name { - if [[ ${ZFS_RPOOL_NAME:-} != "" ]]; then + if [[ -n ${ZFS_RPOOL_NAME:-} ]]; then v_rpool_name=$ZFS_RPOOL_NAME else local rpool_name_invalid_message= @@ -894,7 +894,7 @@ Proceed with the configuration as usual, then, at the partitioning stage: - at the end, choose `Continue Testing` ' - if [[ ${ZFS_NO_INFO_MESSAGES:-} == "" ]]; then + if [[ -z ${ZFS_NO_INFO_MESSAGES:-} ]]; then whiptail --msgbox "$dialog_message" 30 100 fi @@ -939,7 +939,7 @@ Proceed with the configuration as usual, then, at the partitioning stage: - at the end, uncheck `Restart now`, and click `Done` ' - if [[ ${ZFS_NO_INFO_MESSAGES:-} == "" ]]; then + if [[ -z ${ZFS_NO_INFO_MESSAGES:-} ]]; then whiptail --msgbox "$dialog_message" 30 100 fi @@ -1317,7 +1317,7 @@ function update_zed_cache_Debian { # local success= - if [[ ! -s "$c_zfs_mount_dir/etc/zfs/zfs-list.cache/$v_rpool_name" ]]; then + if [[ ! -s $c_zfs_mount_dir/etc/zfs/zfs-list.cache/$v_rpool_name ]]; then # Takes around half second on a test VM. # chroot_execute "zfs set canmount=noauto $v_rpool_name" @@ -1325,7 +1325,7 @@ function update_zed_cache_Debian { SECONDS=0 while [[ $SECONDS -lt 5 ]]; do - if [[ -s "$c_zfs_mount_dir/etc/zfs/zfs-list.cache/$v_rpool_name" ]]; then + if [[ -s $c_zfs_mount_dir/etc/zfs/zfs-list.cache/$v_rpool_name ]]; then success=1 break else @@ -1386,7 +1386,7 @@ function display_exit_banner { You now need to perform a hard reset, then enjoy your ZFS system :-)" - if [[ ${ZFS_NO_INFO_MESSAGES:-} == "" ]]; then + if [[ -z ${ZFS_NO_INFO_MESSAGES:-} ]]; then whiptail --msgbox "$dialog_message" 30 100 fi } @@ -1423,7 +1423,7 @@ invoke "ask_pool_create_options" invoke "install_host_packages" invoke "setup_partitions" -if [[ "${ZFS_OS_INSTALLATION_SCRIPT:-}" == "" ]]; then +if [[ -z ${ZFS_OS_INSTALLATION_SCRIPT:-} ]]; then # Includes the O/S extra configuration, if necessary (network, root pwd, etc.) invoke "install_operating_system" From cc407ebd66461d4b1502970031868bda5f8e1b3f Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Wed, 28 Apr 2021 15:38:59 +0200 Subject: [PATCH 57/84] Cosmetic: Move set_zfs_ppa_requirement forward In preparation for the next changes. --- install-zfs.sh | 82 +++++++++++++++++++++++++------------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index d67bd0d..7ea1c04 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -416,46 +416,6 @@ If you think this is a bug, please open an issue on https://github.com/saveriomi print_variables v_suitable_disks } -# REQUIREMENT: it must be ensured that, for any distro, `apt update` is invoked at this step, as -# subsequent steps rely on it. -# -# There are three parameters: -# -# 1. the tools are preinstalled (ie. Ubuntu Desktop based); -# 2. the default repository supports ZFS 0.8 (ie. Ubuntu 20.04+ based); -# 3. the distro provides the precompiled ZFS module (i.e. Ubuntu based, not Debian) -# -# Fortunately, with Debian-specific logic isolated, we need conditionals based only on #2 - see -# install_host_packages() and install_host_packages_UbuntuServer(). -# -function set_zfs_ppa_requirement { - apt update - - local zfs_package_version - zfs_package_version=$(apt show zfsutils-linux 2> /dev/null | perl -ne 'print /^Version: (\d+\.\d+)/') - - # Test returns true if $zfs_package_version is blank. - # - if [[ ${ZFS_USE_PPA:-} == "1" ]] || dpkg --compare-versions "$zfs_package_version" lt 0.8; then - v_use_ppa=1 - fi -} - -function set_zfs_ppa_requirement_Debian { - # Only update apt; in this case, ZFS packages are handled in a specific way. - - apt update -} - -# Mint 20 has the CDROM repository enabled, but apt fails when updating due to it (or possibly due -# to it being incorrectly setup). -# -function set_zfs_ppa_requirement_Linuxmint { - perl -i -pe 's/^(deb cdrom)/# $1/' /etc/apt/sources.list - - invoke "set_zfs_ppa_requirement" -} - # By using a FIFO, we avoid having to hide statements like `echo $v_passphrase | zpoool create ...` # from the logs. # @@ -511,6 +471,46 @@ export ZFS_FREE_TAIL_SPACE=12 trap _exit_hook EXIT } +# REQUIREMENT: it must be ensured that, for any distro, `apt update` is invoked at this step, as +# subsequent steps rely on it. +# +# There are three parameters: +# +# 1. the tools are preinstalled (ie. Ubuntu Desktop based); +# 2. the default repository supports ZFS 0.8 (ie. Ubuntu 20.04+ based); +# 3. the distro provides the precompiled ZFS module (i.e. Ubuntu based, not Debian) +# +# Fortunately, with Debian-specific logic isolated, we need conditionals based only on #2 - see +# install_host_packages() and install_host_packages_UbuntuServer(). +# +function set_zfs_ppa_requirement { + apt update + + local zfs_package_version + zfs_package_version=$(apt show zfsutils-linux 2> /dev/null | perl -ne 'print /^Version: (\d+\.\d+)/') + + # Test returns true if $zfs_package_version is blank. + # + if [[ ${ZFS_USE_PPA:-} == "1" ]] || dpkg --compare-versions "$zfs_package_version" lt 0.8; then + v_use_ppa=1 + fi +} + +function set_zfs_ppa_requirement_Debian { + # Only update apt; in this case, ZFS packages are handled in a specific way. + + apt update +} + +# Mint 20 has the CDROM repository enabled, but apt fails when updating due to it (or possibly due +# to it being incorrectly setup). +# +function set_zfs_ppa_requirement_Linuxmint { + perl -i -pe 's/^(deb cdrom)/# $1/' /etc/apt/sources.list + + invoke "set_zfs_ppa_requirement" +} + function select_disks { if [[ -n ${ZFS_SELECTED_DISKS:-} ]]; then mapfile -d, -t v_selected_disks < <(echo -n "$ZFS_SELECTED_DISKS") @@ -1406,9 +1406,9 @@ invoke "display_intro_banner" invoke "check_system_memory" invoke "save_disks_log" invoke "find_suitable_disks" -invoke "set_zfs_ppa_requirement" invoke "register_exit_hook" invoke "create_passphrase_named_pipe" +invoke "set_zfs_ppa_requirement" invoke "select_disks" invoke "select_pools_raid_type" From 56f9d0312c3e8fa620c5da26ed784c474432f177 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Wed, 28 Apr 2021 15:41:57 +0200 Subject: [PATCH 58/84] Refactoring: Split apt index update into separate step --- install-zfs.sh | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index 7ea1c04..f24f4fe 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -471,6 +471,19 @@ export ZFS_FREE_TAIL_SPACE=12 trap _exit_hook EXIT } +function update_apt_index { + apt update +} + +# Mint 20 has the CDROM repository enabled, but apt fails when updating due to it (or possibly due +# to it being incorrectly setup). +# +function update_apt_index_Linuxmint { + perl -i -pe 's/^(deb cdrom)/# $1/' /etc/apt/sources.list + + invoke "update_apt_index" +} + # REQUIREMENT: it must be ensured that, for any distro, `apt update` is invoked at this step, as # subsequent steps rely on it. # @@ -484,8 +497,6 @@ export ZFS_FREE_TAIL_SPACE=12 # install_host_packages() and install_host_packages_UbuntuServer(). # function set_zfs_ppa_requirement { - apt update - local zfs_package_version zfs_package_version=$(apt show zfsutils-linux 2> /dev/null | perl -ne 'print /^Version: (\d+\.\d+)/') @@ -498,17 +509,7 @@ function set_zfs_ppa_requirement { function set_zfs_ppa_requirement_Debian { # Only update apt; in this case, ZFS packages are handled in a specific way. - - apt update -} - -# Mint 20 has the CDROM repository enabled, but apt fails when updating due to it (or possibly due -# to it being incorrectly setup). -# -function set_zfs_ppa_requirement_Linuxmint { - perl -i -pe 's/^(deb cdrom)/# $1/' /etc/apt/sources.list - - invoke "set_zfs_ppa_requirement" + : } function select_disks { @@ -1408,6 +1409,7 @@ invoke "save_disks_log" invoke "find_suitable_disks" invoke "register_exit_hook" invoke "create_passphrase_named_pipe" +invoke "update_apt_index" invoke "set_zfs_ppa_requirement" invoke "select_disks" From cbb3e392ed9168c30cb44cff63f60e4b528230eb Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Wed, 28 Apr 2021 15:55:11 +0200 Subject: [PATCH 59/84] Separate step "prepare_standard_repositories", and add universe repository --- install-zfs.sh | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index f24f4fe..c32394c 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -471,22 +471,33 @@ export ZFS_FREE_TAIL_SPACE=12 trap _exit_hook EXIT } -function update_apt_index { - apt update +function prepare_standard_repositories { + # Make sure it's enabled. Ubuntu MATE has it, while the standard Ubuntu doesn't. + # The program exits with success if the repository is already enabled. + # + add-apt-repository --yes --no-update universe } # Mint 20 has the CDROM repository enabled, but apt fails when updating due to it (or possibly due # to it being incorrectly setup). # -function update_apt_index_Linuxmint { +function prepare_standard_repositories_Linuxmint { perl -i -pe 's/^(deb cdrom)/# $1/' /etc/apt/sources.list - invoke "update_apt_index" + # The universe repository may be already enabled, but it's more solid to ensure it. + # + invoke "prepare_standard_repositories" +} + +function prepare_standard_repositories_Debian { + # Debian doesn't require universe (for dialog). + : +} + +function update_apt_index { + apt update } -# REQUIREMENT: it must be ensured that, for any distro, `apt update` is invoked at this step, as -# subsequent steps rely on it. -# # There are three parameters: # # 1. the tools are preinstalled (ie. Ubuntu Desktop based); @@ -1409,6 +1420,7 @@ invoke "save_disks_log" invoke "find_suitable_disks" invoke "register_exit_hook" invoke "create_passphrase_named_pipe" +invoke "prepare_standard_repositories" invoke "update_apt_index" invoke "set_zfs_ppa_requirement" From 1f3c1da2772c7315b87efc720ae65d8c0eb4f297 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Wed, 28 Apr 2021 15:56:14 +0200 Subject: [PATCH 60/84] Rename horrible function `set_zfs_ppa_requirement` --- install-zfs.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index c32394c..ff01de8 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -507,7 +507,7 @@ function update_apt_index { # Fortunately, with Debian-specific logic isolated, we need conditionals based only on #2 - see # install_host_packages() and install_host_packages_UbuntuServer(). # -function set_zfs_ppa_requirement { +function set_use_zfs_ppa { local zfs_package_version zfs_package_version=$(apt show zfsutils-linux 2> /dev/null | perl -ne 'print /^Version: (\d+\.\d+)/') @@ -518,7 +518,7 @@ function set_zfs_ppa_requirement { fi } -function set_zfs_ppa_requirement_Debian { +function set_use_zfs_ppa_Debian { # Only update apt; in this case, ZFS packages are handled in a specific way. : } @@ -1422,7 +1422,7 @@ invoke "register_exit_hook" invoke "create_passphrase_named_pipe" invoke "prepare_standard_repositories" invoke "update_apt_index" -invoke "set_zfs_ppa_requirement" +invoke "set_use_zfs_ppa" invoke "select_disks" invoke "select_pools_raid_type" From b73a961c9430a43af76f345ba5603bd6b372c391 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Wed, 28 Apr 2021 23:41:06 +0200 Subject: [PATCH 61/84] Remove obsolete comment about setting use PPA logic --- install-zfs.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index ff01de8..b7bdea1 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -498,15 +498,6 @@ function update_apt_index { apt update } -# There are three parameters: -# -# 1. the tools are preinstalled (ie. Ubuntu Desktop based); -# 2. the default repository supports ZFS 0.8 (ie. Ubuntu 20.04+ based); -# 3. the distro provides the precompiled ZFS module (i.e. Ubuntu based, not Debian) -# -# Fortunately, with Debian-specific logic isolated, we need conditionals based only on #2 - see -# install_host_packages() and install_host_packages_UbuntuServer(). -# function set_use_zfs_ppa { local zfs_package_version zfs_package_version=$(apt show zfsutils-linux 2> /dev/null | perl -ne 'print /^Version: (\d+\.\d+)/') From ca03c4b7d37e279faed72df9603805b54154c412 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Wed, 28 Apr 2021 23:40:03 +0200 Subject: [PATCH 62/84] Refactored package installation functions KUbuntu doesn't ship with zfsutils-linux installed, however, this doesn't complete the support. --- install-zfs.sh | 65 ++++++++++++++++++++++---------------------------- 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index b7bdea1..2ca6311 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -514,6 +514,16 @@ function set_use_zfs_ppa_Debian { : } +# Whiptail's lack of multiline editing is quite painful. +# +function install_host_base_packages { + # `efibootmgr` needs installation on all the systems. + # the other packages are each required by different distros, so for simplicity, they're all packed + # together. + # + apt install -y efibootmgr dialog software-properties-common +} + function select_disks { if [[ -n ${ZFS_SELECTED_DISKS:-} ]]; then mapfile -d, -t v_selected_disks < <(echo -n "$ZFS_SELECTED_DISKS") @@ -738,7 +748,7 @@ The encryption/mount-related options are automatically added, and must not be sp print_variables v_bpool_create_options v_rpool_create_options } -function install_host_packages { +function install_host_zfs_packages { if [[ $v_use_ppa == "1" ]]; then if [[ ${ZFS_SKIP_LIVE_ZFS_MODULE_INSTALL:-} != "1" ]]; then add-apt-repository --yes "$c_ppa" @@ -757,12 +767,14 @@ function install_host_packages { fi fi - apt install --yes efibootmgr + # Required only by some distros. + # + apt install --yes zfsutils-linux zfs --version > "$c_zfs_module_version_log" 2>&1 } -function install_host_packages_Debian { +function install_host_zfs_packages_Debian { if [[ ${ZFS_SKIP_LIVE_ZFS_MODULE_INSTALL:-} != "1" ]]; then echo "zfs-dkms zfs-dkms/note-incompatible-licenses note true" | debconf-set-selections @@ -775,29 +787,10 @@ function install_host_packages_Debian { modprobe zfs fi - apt install --yes efibootmgr - zfs --version > "$c_zfs_module_version_log" 2>&1 } -# Differently from Ubuntu, Mint doesn't have the package installed in the live version. -# -function install_host_packages_Linuxmint { - apt install --yes zfsutils-linux - - invoke "install_host_packages" -} - -function install_host_packages_elementary { - if [[ ${ZFS_SKIP_LIVE_ZFS_MODULE_INSTALL:-} != "1" ]]; then - apt update - apt install --yes software-properties-common - fi - - invoke "install_host_packages" -} - -function install_host_packages_UbuntuServer { +function install_host_zfs_packages_UbuntuServer { if [[ $v_use_ppa != "1" ]]; then apt install --yes zfsutils-linux efibootmgr @@ -822,7 +815,7 @@ function install_host_packages_UbuntuServer { apt update apt install --yes "linux-headers-$(uname -r)" - install_host_packages + install_host_zfs_packages else apt install --yes efibootmgr fi @@ -1155,7 +1148,13 @@ function prepare_jail { chroot_execute 'echo "nameserver 8.8.8.8" >> /etc/resolv.conf' } -# See install_host_packages() for some comments. +# Same principle as install_host_base_packages(). +# +function install_jail_base_packages { + chroot_execute "apt install --yes rsync grub-efi-amd64-signed shim-signed software-properties-common" +} + +# See install_host_zfs_packages() for some comments. # function install_jail_zfs_packages { if [[ $v_use_ppa == "1" ]]; then @@ -1177,8 +1176,6 @@ function install_jail_zfs_packages { # chroot_execute "apt install --yes libzfs2linux zfs-initramfs zfs-zed zfsutils-linux" fi - - chroot_execute "apt install --yes grub-efi-amd64-signed shim-signed" } function install_jail_zfs_packages_Debian { @@ -1197,18 +1194,12 @@ APT' chroot_execute "apt update" chroot_execute 'echo "zfs-dkms zfs-dkms/note-incompatible-licenses note true" | debconf-set-selections' - chroot_execute "apt install --yes rsync zfs-initramfs zfs-dkms grub-efi-amd64-signed shim-signed" -} - -function install_jail_zfs_packages_elementary { - chroot_execute "apt install --yes software-properties-common" - - invoke "install_jail_zfs_packages" + chroot_execute "apt install --yes zfs-initramfs zfs-dkms" } function install_jail_zfs_packages_UbuntuServer { if [[ $v_use_ppa != "1" ]]; then - chroot_execute "apt install --yes zfsutils-linux zfs-initramfs grub-efi-amd64-signed shim-signed" + chroot_execute "apt install --yes zfsutils-linux zfs-initramfs" else invoke "install_jail_zfs_packages" fi @@ -1414,6 +1405,7 @@ invoke "create_passphrase_named_pipe" invoke "prepare_standard_repositories" invoke "update_apt_index" invoke "set_use_zfs_ppa" +invoke "install_host_base_packages" invoke "select_disks" invoke "select_pools_raid_type" @@ -1425,7 +1417,7 @@ invoke "ask_free_tail_space" invoke "ask_rpool_name" invoke "ask_pool_create_options" -invoke "install_host_packages" +invoke "install_host_zfs_packages" invoke "setup_partitions" if [[ -z ${ZFS_OS_INSTALLATION_SCRIPT:-} ]]; then @@ -1445,6 +1437,7 @@ else fi invoke "prepare_jail" +invoke "install_jail_base_packages" invoke "install_jail_zfs_packages" invoke "prepare_efi_partition" invoke "configure_and_update_grub" From a59fb9ebd4a4fe9d108f87b114deb52692cd593f Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Thu, 29 Apr 2021 00:00:26 +0200 Subject: [PATCH 63/84] Add `--no-update` to all `add-apt-repository` invocations --- install-zfs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index 2ca6311..a8d7e29 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -751,7 +751,7 @@ The encryption/mount-related options are automatically added, and must not be sp function install_host_zfs_packages { if [[ $v_use_ppa == "1" ]]; then if [[ ${ZFS_SKIP_LIVE_ZFS_MODULE_INSTALL:-} != "1" ]]; then - add-apt-repository --yes "$c_ppa" + add-apt-repository --yes --no-update "$c_ppa" apt update # Libelf-dev allows `CONFIG_STACK_VALIDATION` to be set - it's optional, but good to have. @@ -1158,7 +1158,7 @@ function install_jail_base_packages { # function install_jail_zfs_packages { if [[ $v_use_ppa == "1" ]]; then - chroot_execute "add-apt-repository --yes $c_ppa" + chroot_execute "add-apt-repository --yes --no-update $c_ppa" chroot_execute "apt update" From 3fb50c308d778917fea4847e7d74ec0030efaa0f Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Fri, 30 Apr 2021 19:43:15 +0200 Subject: [PATCH 64/84] Fix prepare_standard_repositories_Linuxmint() endless loop --- install-zfs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index a8d7e29..4a16805 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -486,7 +486,7 @@ function prepare_standard_repositories_Linuxmint { # The universe repository may be already enabled, but it's more solid to ensure it. # - invoke "prepare_standard_repositories" + prepare_standard_repositories } function prepare_standard_repositories_Debian { @@ -1201,7 +1201,7 @@ function install_jail_zfs_packages_UbuntuServer { if [[ $v_use_ppa != "1" ]]; then chroot_execute "apt install --yes zfsutils-linux zfs-initramfs" else - invoke "install_jail_zfs_packages" + install_jail_zfs_packages fi } From 1da4b4ce4b4c1b4e764b5226ced540fa69044ce6 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Fri, 30 Apr 2021 19:44:29 +0200 Subject: [PATCH 65/84] Add comment to invoke() about potential mistake --- install-zfs.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install-zfs.sh b/install-zfs.sh index 4a16805..a926320 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -98,6 +98,9 @@ c_udevadm_settle_timeout=10 # seconds # A target function must exist, otherwise a error is raised, unless `--optional` is specified. # `--optional` is useful when a step is specific to a single distribution, e.g. Debian's root password. # +# WATCH OUT! Don't forget *not* to call this from an ovverridden function, otherwise, it will call itself +# endlessly! +# # Examples: # # $ function install_jail_zfs_packages { :; } From 5d34ec853acda8114c0be4e3dd9cb4f06d64c6ba Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Fri, 30 Apr 2021 19:45:50 +0200 Subject: [PATCH 66/84] Fix invoke() example --- install-zfs.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index a926320..09f0e4d 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -105,17 +105,17 @@ c_udevadm_settle_timeout=10 # seconds # # $ function install_jail_zfs_packages { :; } # $ function install_jail_zfs_packages_Debian { :; } -# $ distro_dependent_invoke "install_jail_zfs_packages" +# $ invoke "install_jail_zfs_packages" # # If the distribution is `Debian`, the second will be invoked, otherwise, the first. # # $ function update_zed_cache_Ubuntu { :; } -# $ distro_dependent_invoke "update_zed_cache" --optional +# $ invoke "update_zed_cache" --optional # # If the distribution is `Debian`, nothing will happen. # # $ function update_zed_cache_Ubuntu { :; } -# $ distro_dependent_invoke "update_zed_cache" +# $ invoke "update_zed_cache" # # If the distribution is `Debian`, an error will be raised. # From 3e419f92b02c304c4e9bd36804f372ec29d53bc7 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Fri, 30 Apr 2021 21:43:21 +0200 Subject: [PATCH 67/84] Fix differences between distros in add-apt-repository options Damn Mint! --- install-zfs.sh | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index 09f0e4d..f598bcc 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -195,6 +195,33 @@ function print_variables { echo } +# Very annoyingly, some distros (e.g. Linux Mint) have a "reduced" version of add-apt-repository +# +function checked_add_apt_repository { + local repository=$1 + local option=${2:-} # optional: `--chroot` + + local add_repo_command=(add-apt-repository --yes "$repository") + + if add-apt-repository --help | grep -q "\--no-update"; then + # Assume that when this option isn't available, no update is performed. The fragmentation is a PITA. + # + add_repo_command+=(--no-update) + fi + + case $option in + '') + "${add_repo_command[@]}" + ;; + --chroot) + chroot_execute "${add_repo_command[*]}" + ;; + *) + >&2 echo "Unexpected checked_add_apt_repository option: $2" + exit 1 + esac +} + function chroot_execute { chroot $c_zfs_mount_dir bash -c "$1" } @@ -478,7 +505,7 @@ function prepare_standard_repositories { # Make sure it's enabled. Ubuntu MATE has it, while the standard Ubuntu doesn't. # The program exits with success if the repository is already enabled. # - add-apt-repository --yes --no-update universe + checked_add_apt_repository universe } # Mint 20 has the CDROM repository enabled, but apt fails when updating due to it (or possibly due @@ -754,7 +781,7 @@ The encryption/mount-related options are automatically added, and must not be sp function install_host_zfs_packages { if [[ $v_use_ppa == "1" ]]; then if [[ ${ZFS_SKIP_LIVE_ZFS_MODULE_INSTALL:-} != "1" ]]; then - add-apt-repository --yes --no-update "$c_ppa" + checked_add_apt_repository "$c_ppa" apt update # Libelf-dev allows `CONFIG_STACK_VALIDATION` to be set - it's optional, but good to have. @@ -1161,7 +1188,7 @@ function install_jail_base_packages { # function install_jail_zfs_packages { if [[ $v_use_ppa == "1" ]]; then - chroot_execute "add-apt-repository --yes --no-update $c_ppa" + checked_add_apt_repository "$c_ppa" --chroot chroot_execute "apt update" From b0d7ee81229fbb2678c9d15af17a4c72e8e87398 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 5 Jun 2021 22:44:56 +0200 Subject: [PATCH 68/84] Ubuntu Server: Create `/target` if missing The directory used to be created before [20.04.2]... AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARGH!!!!!!!!!! --- install-zfs.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/install-zfs.sh b/install-zfs.sh index f598bcc..49e3da4 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1034,6 +1034,12 @@ You can switch anytime to this terminal, and back, in order to read the instruct # Server, but it's better not to take risks. # if ! mountpoint -q "$c_installed_os_mount_dir"; then + # There must be a conspiracy 🙄 `/target` used to be created before [20.04.2]. + # + if [[ ! -d $c_installed_os_mount_dir ]]; then + mkdir "$c_installed_os_mount_dir" + fi + mount "${v_temp_volume_device}p2" "$c_installed_os_mount_dir" fi From 759f6067ee4d733bcb143993804775f7454c0e9c Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 19:18:10 +0200 Subject: [PATCH 69/84] Update pool creation features to updated procedure --- install-zfs.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/install-zfs.sh b/install-zfs.sh index 49e3da4..da5d104 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -52,7 +52,24 @@ c_memory_warning_limit=$((3584 - 128)) # megabytes; exclude some RAM, which can c_default_bpool_create_options=( -o ashift=12 -o autotrim=on + -d + -o feature@async_destroy=enabled + -o feature@bookmarks=enabled + -o feature@embedded_data=enabled + -o feature@empty_bpobj=enabled + -o feature@enabled_txg=enabled + -o feature@extensible_dataset=enabled + -o feature@filesystem_limits=enabled + -o feature@hole_birth=enabled + -o feature@large_blocks=enabled + -o feature@lz4_compress=enabled + -o feature@spacemap_histogram=enabled + -O acltype=posixacl + -O compression=lz4 -O devices=off + -O normalization=formD + -O relatime=on + -O xattr=sa ) c_default_rpool_create_options=( -o ashift=12 From 8b6059752fac89055d0ec29b441f9521da86d41a Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Mon, 14 Jun 2021 11:34:33 +0200 Subject: [PATCH 70/84] Fix /boot mounts mess (part 1: timeout) --- install-zfs.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index da5d104..1afbda9 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1259,7 +1259,10 @@ function install_jail_zfs_packages_UbuntuServer { } function prepare_efi_partition { - chroot_execute "echo PARTUUID=$(blkid -s PARTUUID -o value "${v_selected_disks[0]}-part1") /boot/efi vfat nofail,x-systemd.device-timeout=1 0 1 > /etc/fstab" + # It's important to give a long enough timeout time; on relatively slow machines (e.g. virtual machines), + # a timeout of 1 will fail. + # + chroot_execute "echo PARTUUID=$(blkid -s PARTUUID -o value "${v_selected_disks[0]}-part1") /boot/efi vfat nofail,x-systemd.device-timeout=10 0 1 > /etc/fstab" chroot_execute "mkdir -p /boot/efi" chroot_execute "mount /boot/efi" @@ -1299,7 +1302,9 @@ function sync_efi_partitions { for ((i = 1; i < ${#v_selected_disks[@]}; i++)); do local synced_efi_partition_path="/boot/efi$((i + 1))" - chroot_execute "echo PARTUUID=$(blkid -s PARTUUID -o value "${v_selected_disks[i]}-part1") $synced_efi_partition_path vfat nofail,x-systemd.device-timeout=1 0 1 >> /etc/fstab" + # Reference: prepare_efi_partition() + # + chroot_execute "echo PARTUUID=$(blkid -s PARTUUID -o value "${v_selected_disks[i]}-part1") $synced_efi_partition_path vfat nofail,x-systemd.device-timeout=10 0 1 >> /etc/fstab" chroot_execute "mkdir -p $synced_efi_partition_path" chroot_execute "mount $synced_efi_partition_path" From dfdc26e0cef71f1b1c9bb3900ddf3c4c52d24972 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Mon, 14 Jun 2021 12:25:11 +0200 Subject: [PATCH 71/84] Fix /boot mounts mess (part 2: /boot dependency) --- install-zfs.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index 1afbda9..46f2774 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1259,10 +1259,12 @@ function install_jail_zfs_packages_UbuntuServer { } function prepare_efi_partition { + # Wait for /boot to be mounted before mounting the EFI partitions; we use the services as dependency + # rather than a mount (`requires-mounts-for`) because the `/boot` path may exist but still not mounted. # It's important to give a long enough timeout time; on relatively slow machines (e.g. virtual machines), # a timeout of 1 will fail. # - chroot_execute "echo PARTUUID=$(blkid -s PARTUUID -o value "${v_selected_disks[0]}-part1") /boot/efi vfat nofail,x-systemd.device-timeout=10 0 1 > /etc/fstab" + chroot_execute "echo PARTUUID=$(blkid -s PARTUUID -o value "${v_selected_disks[0]}-part1") /boot/efi vfat nofail,x-systemd.requires=zfs-mount.service,x-systemd.device-timeout=10 0 1 > /etc/fstab" chroot_execute "mkdir -p /boot/efi" chroot_execute "mount /boot/efi" @@ -1304,7 +1306,7 @@ function sync_efi_partitions { # Reference: prepare_efi_partition() # - chroot_execute "echo PARTUUID=$(blkid -s PARTUUID -o value "${v_selected_disks[i]}-part1") $synced_efi_partition_path vfat nofail,x-systemd.device-timeout=10 0 1 >> /etc/fstab" + chroot_execute "echo PARTUUID=$(blkid -s PARTUUID -o value "${v_selected_disks[i]}-part1") $synced_efi_partition_path vfat nofail,x-systemd.requires=zfs-mount.service,x-systemd.device-timeout=10 0 1 >> /etc/fstab" chroot_execute "mkdir -p $synced_efi_partition_path" chroot_execute "mount $synced_efi_partition_path" From 8f2046c88efd5364a09cda03f6b4567bdd6f183c Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 22:48:41 +0200 Subject: [PATCH 72/84] Use UUID rather than PARTUUID for boot partitions in the fstab Should be essentially the same, but this is the new procedure. --- install-zfs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index 46f2774..9cfef27 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1264,7 +1264,7 @@ function prepare_efi_partition { # It's important to give a long enough timeout time; on relatively slow machines (e.g. virtual machines), # a timeout of 1 will fail. # - chroot_execute "echo PARTUUID=$(blkid -s PARTUUID -o value "${v_selected_disks[0]}-part1") /boot/efi vfat nofail,x-systemd.requires=zfs-mount.service,x-systemd.device-timeout=10 0 1 > /etc/fstab" + chroot_execute "echo /dev/disk/by-uuid/$(blkid -s UUID -o value "${v_selected_disks[0]}"-part1) /boot/efi vfat nofail,x-systemd.requires=zfs-mount.service,x-systemd.device-timeout=10 0 1 > /etc/fstab" chroot_execute "mkdir -p /boot/efi" chroot_execute "mount /boot/efi" @@ -1306,7 +1306,7 @@ function sync_efi_partitions { # Reference: prepare_efi_partition() # - chroot_execute "echo PARTUUID=$(blkid -s PARTUUID -o value "${v_selected_disks[i]}-part1") $synced_efi_partition_path vfat nofail,x-systemd.requires=zfs-mount.service,x-systemd.device-timeout=10 0 1 >> /etc/fstab" + chroot_execute "echo /dev/disk/by-uuid/$(blkid -s UUID -o value "${v_selected_disks[i]}"-part1) $synced_efi_partition_path vfat nofail,x-systemd.requires=zfs-mount.service,x-systemd.device-timeout=10 0 1 >> /etc/fstab" chroot_execute "mkdir -p $synced_efi_partition_path" chroot_execute "mount $synced_efi_partition_path" From b7ed3583425198a8fbb4981fa9bf29bbea967aab Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Mon, 14 Jun 2021 13:18:39 +0200 Subject: [PATCH 73/84] Refactoring: Move fstab preparation into separate step --- install-zfs.sh | 45 ++++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index 9cfef27..f132cb7 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1258,14 +1258,34 @@ function install_jail_zfs_packages_UbuntuServer { fi } -function prepare_efi_partition { - # Wait for /boot to be mounted before mounting the EFI partitions; we use the services as dependency - # rather than a mount (`requires-mounts-for`) because the `/boot` path may exist but still not mounted. - # It's important to give a long enough timeout time; on relatively slow machines (e.g. virtual machines), - # a timeout of 1 will fail. - # - chroot_execute "echo /dev/disk/by-uuid/$(blkid -s UUID -o value "${v_selected_disks[0]}"-part1) /boot/efi vfat nofail,x-systemd.requires=zfs-mount.service,x-systemd.device-timeout=10 0 1 > /etc/fstab" +function prepare_fstab { + chroot_execute "true > /etc/fstab" + for ((i = 0; i < ${#v_selected_disks[@]}; i++)); do + if (( i == 0 )); then + local mountpoint=/boot/efi + else + local mountpoint=/boot/efi$((i + 1)) + fi + + # Wait for /boot to be mounted before mounting the EFI partitions; we use the services as dependency + # rather than a mount (`requires-mounts-for`) because the `/boot` path may exist but still not mounted. + # It's important to give a long enough timeout time; on relatively slow machines (e.g. virtual machines), + # a timeout of 1 will fail. + # + chroot_execute "echo /dev/disk/by-uuid/$(blkid -s UUID -o value "${v_selected_disks[i]}"-part1) $mountpoint vfat nofail,x-systemd.requires=zfs-mount.service,x-systemd.device-timeout=10 0 1 >> /etc/fstab" + done + + # The service is created in the configure_boot_pool_import() step. + # + chroot_execute "echo $c_bpool_name /boot zfs nodev,relatime,x-systemd.requires=zfs-import-$c_bpool_name.service 0 0 >> /etc/fstab" + + if (( v_swap_size > 0 )); then + chroot_execute "echo /dev/zvol/$v_rpool_name/swap none swap discard 0 0 >> /etc/fstab" + fi +} + +function prepare_efi_partition { chroot_execute "mkdir -p /boot/efi" chroot_execute "mount /boot/efi" @@ -1304,10 +1324,6 @@ function sync_efi_partitions { for ((i = 1; i < ${#v_selected_disks[@]}; i++)); do local synced_efi_partition_path="/boot/efi$((i + 1))" - # Reference: prepare_efi_partition() - # - chroot_execute "echo /dev/disk/by-uuid/$(blkid -s UUID -o value "${v_selected_disks[i]}"-part1) $synced_efi_partition_path vfat nofail,x-systemd.requires=zfs-mount.service,x-systemd.device-timeout=10 0 1 >> /etc/fstab" - chroot_execute "mkdir -p $synced_efi_partition_path" chroot_execute "mount $synced_efi_partition_path" @@ -1342,7 +1358,8 @@ UNIT" chroot_execute "systemctl enable zfs-import-$c_bpool_name.service" chroot_execute "zfs set mountpoint=legacy $c_bpool_name" - chroot_execute "echo $c_bpool_name /boot zfs nodev,relatime,x-systemd.requires=zfs-import-$c_bpool_name.service 0 0 >> /etc/fstab" + + # the fstab entry is added in the prepare_fstab() step. } # This step is important in cases where the keyboard layout is not the standard one. @@ -1399,7 +1416,8 @@ function update_zed_cache_Debian { } function configure_remaining_settings { - [[ $v_swap_size -gt 0 ]] && chroot_execute "echo /dev/zvol/$v_rpool_name/swap none swap discard 0 0 >> /etc/fstab" || true + # The swap volume, if specified, is set in the prepare_fstab() step. + chroot_execute "echo RESUME=none > /etc/initramfs-tools/conf.d/resume" } @@ -1499,6 +1517,7 @@ fi invoke "prepare_jail" invoke "install_jail_base_packages" invoke "install_jail_zfs_packages" +invoke "prepare_fstab" invoke "prepare_efi_partition" invoke "configure_and_update_grub" invoke "sync_efi_partitions" From ddb4d1532ad7c39f64ba846ffc824cba069cf601 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Mon, 14 Jun 2021 13:49:32 +0200 Subject: [PATCH 74/84] Refactoring: De-inline conditional command in fix_filesystem_mount_ordering() --- install-zfs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-zfs.sh b/install-zfs.sh index f132cb7..71208fa 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1375,7 +1375,7 @@ function update_zed_cache_Debian { # On Debian, this file may exist already. # - chroot_execute "[[ ! -f /etc/zfs/zed.d/history_event-zfs-list-cacher.sh ]] && ln -s /usr/lib/zfs-linux/zed.d/history_event-zfs-list-cacher.sh /etc/zfs/zed.d/" + chroot_execute "if [[ ! -f /etc/zfs/zed.d/history_event-zfs-list-cacher.sh ]]; then ln -s /usr/lib/zfs-linux/zed.d/history_event-zfs-list-cacher.sh /etc/zfs/zed.d/; fi" # Assumed to be present by the zedlet above, but missing. # Filed issue: https://github.com/zfsonlinux/zfs/issues/9945. From 4c34dcb19497e2572c4cf8195e456bd2aa3fa56f Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 6 Jun 2021 11:57:57 +0200 Subject: [PATCH 75/84] Generalize fix filesystem mount ordering Previously, it wasn't taken into account that the root FS may not be `$v_rpool_name/ROOT`. --- install-zfs.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/install-zfs.sh b/install-zfs.sh index 71208fa..0d1422c 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1389,9 +1389,15 @@ function update_zed_cache_Debian { local success= if [[ ! -s $c_zfs_mount_dir/etc/zfs/zfs-list.cache/$v_rpool_name ]]; then + local zfs_root_fs zfs_boot_fs + + zfs_root_fs=$(chroot_execute 'zfs list / | awk "NR==2 {print \$1}"') + zfs_boot_fs=$(chroot_execute 'zfs list /boot | awk "NR==2 {print \$1}"') + # Takes around half second on a test VM. # - chroot_execute "zfs set canmount=noauto $v_rpool_name" + chroot_execute "zfs set canmount=on $zfs_boot_fs" + chroot_execute "zfs set canmount=on $zfs_root_fs" SECONDS=0 From 44f12dce597caa5bb74f036714964710cdf2e173 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Mon, 14 Jun 2021 14:15:55 +0200 Subject: [PATCH 76/84] Improve old update_zed_cache_Debian(), following the new procedure --- install-zfs.sh | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index 0d1422c..de56a46 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1369,32 +1369,34 @@ function update_initramfs { chroot_execute "update-initramfs -u" } -function update_zed_cache_Debian { +function fix_filesystem_mount_ordering_Debian { chroot_execute "mkdir /etc/zfs/zfs-list.cache" - chroot_execute "touch /etc/zfs/zfs-list.cache/$v_rpool_name" + chroot_execute "touch /etc/zfs/zfs-list.cache/$c_bpool_name /etc/zfs/zfs-list.cache/$v_rpool_name" # On Debian, this file may exist already. # chroot_execute "if [[ ! -f /etc/zfs/zed.d/history_event-zfs-list-cacher.sh ]]; then ln -s /usr/lib/zfs-linux/zed.d/history_event-zfs-list-cacher.sh /etc/zfs/zed.d/; fi" - # Assumed to be present by the zedlet above, but missing. + # Assumed to be present by the zedlet above on Debian, but missing. # Filed issue: https://github.com/zfsonlinux/zfs/issues/9945. # chroot_execute "mkdir /run/lock" + # It's not clear (based on the help) why it's explicitly run in foreground (`-F`), but backgrounded. + # chroot_execute "zed -F &" # We could pool the events via `zpool events -v`, but it's much simpler to just check on the file. # local success= - if [[ ! -s $c_zfs_mount_dir/etc/zfs/zfs-list.cache/$v_rpool_name ]]; then + if [[ ! -s $c_zfs_mount_dir/etc/zfs/zfs-list.cache/$c_bpool_name || ! -s $c_zfs_mount_dir/etc/zfs/zfs-list.cache/$v_rpool_name ]]; then local zfs_root_fs zfs_boot_fs zfs_root_fs=$(chroot_execute 'zfs list / | awk "NR==2 {print \$1}"') zfs_boot_fs=$(chroot_execute 'zfs list /boot | awk "NR==2 {print \$1}"') - # Takes around half second on a test VM. + # For the rpool only, it takes around half second on a test VM. # chroot_execute "zfs set canmount=on $zfs_boot_fs" chroot_execute "zfs set canmount=on $zfs_root_fs" @@ -1402,23 +1404,25 @@ function update_zed_cache_Debian { SECONDS=0 while [[ $SECONDS -lt 5 ]]; do - if [[ -s $c_zfs_mount_dir/etc/zfs/zfs-list.cache/$v_rpool_name ]]; then + if [[ ! -s $c_zfs_mount_dir/etc/zfs/zfs-list.cache/$c_bpool_name || ! -s $c_zfs_mount_dir/etc/zfs/zfs-list.cache/$v_rpool_name ]]; then success=1 break else sleep 0.25 fi done + else + success=1 fi + chroot_execute "pkill zed" + if [[ $success -ne 1 ]]; then echo "Error: The ZFS cache hasn't been updated by ZED!" exit 1 fi - chroot_execute "pkill zed" - - chroot_execute "sed -Ei 's|$c_installed_os_mount_dir/?|/|' /etc/zfs/zfs-list.cache/$v_rpool_name" + chroot_execute "sed -Ei 's|$c_zfs_mount_dir/?|/|' /etc/zfs/zfs-list.cache/*" } function configure_remaining_settings { @@ -1529,7 +1533,7 @@ invoke "configure_and_update_grub" invoke "sync_efi_partitions" invoke "configure_boot_pool_import" invoke "update_initramfs" -invoke "update_zed_cache" --optional +invoke "fix_filesystem_mount_ordering" --optional invoke "configure_remaining_settings" invoke "prepare_for_system_exit" From 9126c2392672f1d4dc0b22eadaf993cdaa18a119 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sat, 24 Apr 2021 09:38:43 +0200 Subject: [PATCH 77/84] Replace Travis with GitHub actions --- .github/workflows/ci.yml | 11 +++++++++++ .travis.yml | 8 -------- 2 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e19289c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,11 @@ +name: CI + +on: pull_request + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Run shellcheck + run: ci/run_shellcheck.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 27cee52..0000000 --- a/.travis.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -dist: bionic -language: bash -cache: - directories: - - "/opt/shellcheck" -script: -- ci/run_shellcheck From 3a1bcda1aa19f01dde70c64b58fa3f7302a4cdb1 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Tue, 27 Apr 2021 11:12:40 +0200 Subject: [PATCH 78/84] Add support for datasets --- install-zfs.sh | 73 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 67 insertions(+), 6 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index de56a46..808507d 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -82,6 +82,34 @@ c_default_rpool_create_options=( -O xattr=sa -O devices=off ) +# Can't include double quotes, due to the templating logic. +# +c_default_dataset_create_options=' +ROOT mountpoint=/ com.ubuntu.zsys:bootfs=yes com.ubuntu.zsys:last-used=$(date +%s) +ROOT/srv com.ubuntu.zsys:bootfs=no +ROOT/usr canmount=off com.ubuntu.zsys:bootfs=no +ROOT/usr/local +ROOT/var canmount=off com.ubuntu.zsys:bootfs=no +ROOT/var/games +ROOT/var/lib +ROOT/var/lib/AccountsService +ROOT/var/lib/apt +ROOT/var/lib/dpkg +ROOT/var/lib/NetworkManager +ROOT/var/log +ROOT/var/mail +ROOT/var/snap +ROOT/var/spool +ROOT/var/www +ROOT/tmp com.ubuntu.zsys:bootfs=no + +USERDATA mountpoint=/ canmount=off +USERDATA/root mountpoint=/root canmount=on com.ubuntu.zsys:bootfs-datasets=$v_rpool_name/ROOT + +$(find $c_installed_os_mount_dir/home -mindepth 1 -maxdepth 1 -printf '\'' +USERDATA/%P mountpoint=/home/%P canmount=on com.ubuntu.zsys:bootfs-datasets=$v_rpool_name/%P +'\'') +' c_zfs_mount_dir=/mnt c_installed_os_mount_dir=/target declare -A c_supported_linux_distributions=([Ubuntu]="18.04 20.04" [UbuntuServer]="18.04 20.04" [LinuxMint]="19.1 19.2 19.3" [Linuxmint]="20 20.1" [elementary]=5.1) @@ -1067,7 +1095,7 @@ function custom_install_operating_system { sudo "$ZFS_OS_INSTALLATION_SCRIPT" } -function create_pools { +function create_pools_and_datasets { # POOL OPTIONS ####################### local encryption_options=() @@ -1085,9 +1113,10 @@ function create_pools { bpool_disks_partitions+=("${selected_disk}-part2") done - # POOLS CREATION ##################### + # ROOT POOL CREATION ################# - # The root pool must be created first, since the boot pool mountpoint is inside it. + # In this script, the boot pool doesn't have a root dataset; since its creation will mount /boot, + # it needs to be done after the root pool root dataset is created. set +x echo -n "$v_passphrase" > "$c_passphrase_named_pipe" & @@ -1102,10 +1131,42 @@ function create_pools { zpool create \ "${encryption_options[@]}" \ "${v_rpool_create_options[@]}" \ - -O mountpoint=/ -R "$c_zfs_mount_dir" -f \ + -O mountpoint=/ -O canmount=off -R "$c_zfs_mount_dir" -f \ "$v_rpool_name" "${v_pools_raid_type[@]}" "${rpool_disks_partitions[@]}" \ < "$c_passphrase_named_pipe" + # DATASETS CREATION ################## + + local interpolated_dataset_create_options + interpolated_dataset_create_options=$(eval echo \""$c_default_dataset_create_options"\") + + echo "Interpolated dataset create options:" + echo "$interpolated_dataset_create_options" + echo + + while read -r dataset_metadata_line || [[ -n $dataset_metadata_line ]]; do + if [[ $dataset_metadata_line =~ [^[:space:]] ]]; then + local dataset_metadata_entries + # shellcheck disable=2206 # cheating for simplicity (alternative: sed and mapfile). + dataset_metadata_entries=($dataset_metadata_line) + + local dataset=$v_rpool_name/${dataset_metadata_entries[0]} + local options=("${dataset_metadata_entries[@]:1}") + + # Prepend the `-o`. + # + # shellcheck disable=2068 # cheating for simplicity (otherwise each `-o $option` will be a single + # string). + zfs create ${options[@]/#/-o } "$dataset" + fi + done < <(echo "$interpolated_dataset_create_options") + + chmod 700 /mnt/root + # This is fine independently of the user creating a dataset for /tmp or not. + chmod 1777 /mnt/tmp + + # BOOT POOL CREATION ################# + zpool create \ "${v_bpool_create_options[@]}" \ -O mountpoint=/boot -R "$c_zfs_mount_dir" -f \ @@ -1512,12 +1573,12 @@ if [[ -z ${ZFS_OS_INSTALLATION_SCRIPT:-} ]]; then # Includes the O/S extra configuration, if necessary (network, root pwd, etc.) invoke "install_operating_system" - invoke "create_pools" + invoke "create_pools_and_datasets" invoke "create_swap_volume" invoke "sync_os_temp_installation_dir_to_rpool" invoke "remove_temp_partition_and_expand_rpool" else - invoke "create_pools" + invoke "create_pools_and_datasets" invoke "create_swap_volume" invoke "remove_temp_partition_and_expand_rpool" From faf722d89c4713028a1e71e2a67ff4a4f504dffc Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Tue, 27 Apr 2021 18:46:13 +0200 Subject: [PATCH 79/84] Allow dataset creation options customization via env variable --- install-zfs.sh | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/install-zfs.sh b/install-zfs.sh index 808507d..8ad78b4 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -25,6 +25,7 @@ v_passphrase= v_root_password= # Debian-only v_rpool_name= v_rpool_create_options= # array; see defaults below for format +v_dataset_create_options= # string; see help for format v_pools_raid_type=() declare -a v_selected_disks # (/dev/by-id/disk_id, ...) v_swap_size= # integer @@ -82,6 +83,10 @@ c_default_rpool_create_options=( -O xattr=sa -O devices=off ) +c_dataset_options_help='# The defaults create a root pool similar to the Ubuntu default; see the script help for details. +# Double quotes are forbidden; lines starting with a hash (`#`) are ignored. +# Parameters and command substitutions are applied; useful variables are $c_zfs_mount_dir and $v_rpool_name. +' # Can't include double quotes, due to the templating logic. # c_default_dataset_create_options=' @@ -293,6 +298,7 @@ The procedure can be entirely automated via environment variables: - ZFS_BPOOL_CREATE_OPTIONS : boot pool options to set on creation (see defaults below) - ZFS_RPOOL_CREATE_OPTIONS : root pool options to set on creation (see defaults below) - ZFS_POOLS_RAID_TYPE : options: blank (striping), `mirror`, `raidz`, `raidz2`, `raidz3`; if unset, it will be asked. +- ZFS_DATASET_CREATE_OPTIONS : see explanation below - ZFS_NO_INFO_MESSAGES : set 1 to skip informational messages - ZFS_SWAP_SIZE : swap size (integer); set 0 for no swap - ZFS_FREE_TAIL_SPACE : leave free space at the end of each disk (integer), for example, for a swap partition @@ -308,6 +314,19 @@ When installing the O/S via $ZFS_OS_INSTALLATION_SCRIPT, the root pool is mounte Boot pool default create options: '"${c_default_bpool_create_options[*]/#-/$'\n' -}"' Root pool default create options: '"${c_default_rpool_create_options[*]/#-/$'\n' -}"' + +The root pool dataset creation options can be specified by passing a string of whom each line has: + +- the dataset name (without the pool) +- the options (without `-o`) + +The defaults, which create a root pool similar to the Ubuntu default, are: + +'"$(echo -n "$c_default_dataset_create_options" | sed 's/^/ /')"' + +Double quotes are forbidden. Parameters and command substitutions are applied; useful variables are $c_zfs_mount_dir and $v_rpool_name. + +Datasets are created after the operating system is installed; at that stage, it'\'' mounted in the directory specified by $c_zfs_mount_dir. ' echo "$help" @@ -823,6 +842,29 @@ The encryption/mount-related options are automatically added, and must not be sp print_variables v_bpool_create_options v_rpool_create_options } +function ask_dataset_create_options { + if [[ -n ${ZFS_DATASET_CREATE_OPTIONS:-} ]]; then + v_dataset_create_options=$ZFS_DATASET_CREATE_OPTIONS + else + while true; do + local tempfile + tempfile=$(mktemp) + + echo "$c_dataset_options_help$c_default_dataset_create_options" > "$tempfile" + + local user_value + user_value=$(dialog --editbox "$tempfile" 30 120 3>&1 1>&2 2>&3) + + if [[ -n $user_value && $user_value != *\"* ]]; then + v_dataset_create_options=$(echo "$user_value" | perl -ne 'print unless /^\s*#/') + break + fi + done + fi + + print_variables v_dataset_create_options +} + function install_host_zfs_packages { if [[ $v_use_ppa == "1" ]]; then if [[ ${ZFS_SKIP_LIVE_ZFS_MODULE_INSTALL:-} != "1" ]]; then @@ -1138,7 +1180,7 @@ function create_pools_and_datasets { # DATASETS CREATION ################## local interpolated_dataset_create_options - interpolated_dataset_create_options=$(eval echo \""$c_default_dataset_create_options"\") + interpolated_dataset_create_options=$(eval echo \""$v_dataset_create_options"\") echo "Interpolated dataset create options:" echo "$interpolated_dataset_create_options" @@ -1565,6 +1607,7 @@ invoke "ask_swap_size" invoke "ask_free_tail_space" invoke "ask_rpool_name" invoke "ask_pool_create_options" +invoke "ask_dataset_create_options" invoke "install_host_zfs_packages" invoke "setup_partitions" From c7c0465a67b3d262ac2d689919b342b38e76afd7 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Tue, 27 Apr 2021 19:54:09 +0200 Subject: [PATCH 80/84] Update README --- README.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b036e60..b4e297d 100644 --- a/README.md +++ b/README.md @@ -54,19 +54,14 @@ As of 20.04, Canonical makes available an experimental ZFS installer on Ubuntu D The advantages of this project over the Ubuntu installer are: -1. it supports pools configuration; -1. it allows specifying the RAID type; -1. it allows customization of the disk partitions; -1. it supports new OpenZFS versions, via PPA `jonathonf/zfs`. +1. it allows configuring pools, datasets and the RAID type; +1. it allows customizing the disk partitions; 1. it supports additional features (e.g. encryption and trimming); +1. it supports newer OpenZFS versions, via PPA `jonathonf/zfs`. 1. it supports many more operating systems; 1. it supports unattended installations, via custom scripts; 1. it's easy to extend. -The disadvantages are: - -1. the Ubuntu installer has a more sophisticated filesystem layout - it separates base directories into different ZFS filesystems (this is planned to be implemented in the ZFS installer as well). - ## Instructions Start the live CD of a supported Linux distribution, then open a terminal and execute: From 7a626226e6e442112593872ea027617f0123b0df Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Thu, 29 Apr 2021 00:12:11 +0200 Subject: [PATCH 81/84] Complete KUbuntu support --- install-zfs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install-zfs.sh b/install-zfs.sh index 8ad78b4..e90b952 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -88,6 +88,7 @@ c_dataset_options_help='# The defaults create a root pool similar to the Ubuntu # Parameters and command substitutions are applied; useful variables are $c_zfs_mount_dir and $v_rpool_name. ' # Can't include double quotes, due to the templating logic. +# KUbuntu has a /home/.directory, which must not be a separate dataset (it's a symlink). # c_default_dataset_create_options=' ROOT mountpoint=/ com.ubuntu.zsys:bootfs=yes com.ubuntu.zsys:last-used=$(date +%s) @@ -111,7 +112,7 @@ ROOT/tmp com.ubuntu.zsys:bootfs=no USERDATA mountpoint=/ canmount=off USERDATA/root mountpoint=/root canmount=on com.ubuntu.zsys:bootfs-datasets=$v_rpool_name/ROOT -$(find $c_installed_os_mount_dir/home -mindepth 1 -maxdepth 1 -printf '\'' +$(find $c_installed_os_mount_dir/home -mindepth 1 -maxdepth 1 -not -name '\''.*'\'' -printf '\'' USERDATA/%P mountpoint=/home/%P canmount=on com.ubuntu.zsys:bootfs-datasets=$v_rpool_name/%P '\'') ' From a94f589f3b584ae1e4ccbd63e1dd3699782e7029 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Sun, 13 Jun 2021 10:36:35 +0200 Subject: [PATCH 82/84] Don't set the permissions on rpool directories See the comment. This was also a bug, if the user was not creating `/root` or `/tmp` filesystems. --- install-zfs.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/install-zfs.sh b/install-zfs.sh index e90b952..cf91c3e 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -1204,9 +1204,10 @@ function create_pools_and_datasets { fi done < <(echo "$interpolated_dataset_create_options") - chmod 700 /mnt/root - # This is fine independently of the user creating a dataset for /tmp or not. - chmod 1777 /mnt/tmp + # Here, the original procedure sets the permissions for /root (700) and /tmp (1777), however, in this + # script we don't need to do it, since we sync the O/S installer result, which is already configured. + # In case of changes, don't forget that the destination layout may be empty, at this point, due to + # the user's ZFS filesystems configuration! # BOOT POOL CREATION ################# From cf72ba07a0e564a17e9913c8abff35bbfaa060e4 Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Mon, 14 Jun 2021 15:38:28 +0200 Subject: [PATCH 83/84] Revert "Remove official support for Debian" This reverts commit 89964d9bc2aed8f4cf3f4040c01b20b8bf364cfe. --- README.md | 5 +++-- install-zfs.sh | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b4e297d..f88e1c6 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # zfs-installer -ZFS installer is a shell script program that fully prepares ZFS on a system, and allows an effortless installation of several Ubuntu-based operating systems using their standard installer (or debootstrap, or any custom script). +ZFS installer is a shell script program that fully prepares ZFS on a system, and allows an effortless installation of several Debian-based operating systems using their standard installer (or debootstrap, or any custom script). - [Status](#status) - [Requirements and functionality](#requirements-and-functionality) @@ -38,6 +38,7 @@ The program currently supports: - Ubuntu Desktop 18.04.x/20.04 Live - Ubuntu Server 18.04.x/20.04 Live - Linux Mint 19.x, 20 +- Debian 10.x Live (desktop environment required) - ElementaryOS 5.1 The ZFS version installed is 0.8 (optionally, 2.x), which supports native encryption and trimming (among the other improvements over 0.7). The required repositories are automatically added to the destination system. @@ -46,7 +47,7 @@ EFI boot is required (any modern (2011+) system will do); legacy boot is current All the ZFS RAID types are supported, with any arbitrary number of disks. An EFI partition is created on each disk, for redundancy purposes. -It's fairly easy to extend the program to support other Ubuntu-based operating systems - the project is open to feature requests. +It's fairly easy to extend the program to support other Debian-based operating systems (e.g. older/newer Ubuntu's, etc.) - the project is (very) open to feature requests. ## Comparison with Ubuntu built-in installer diff --git a/install-zfs.sh b/install-zfs.sh index cf91c3e..03c69f9 100755 --- a/install-zfs.sh +++ b/install-zfs.sh @@ -33,7 +33,7 @@ v_free_tail_space= # integer # Variables set during execution -v_linux_distribution= # Ubuntu, LinuxMint, ... WATCH OUT: not necessarily from `lsb_release` (ie. UbuntuServer) +v_linux_distribution= # Debian, Ubuntu, ... WATCH OUT: not necessarily from `lsb_release` (ie. UbuntuServer) v_use_ppa= # 1=true, false otherwise (applies only to Ubuntu-based). v_temp_volume_device= # /dev/zdN; scope: setup_partitions -> sync_os_temp_installation_dir_to_rpool v_suitable_disks=() # (/dev/by-id/disk_id, ...); scope: find_suitable_disks -> select_disk @@ -118,7 +118,7 @@ USERDATA/%P mountpoint=/home/%P canmount=on com.ubuntu.zsys:b ' c_zfs_mount_dir=/mnt c_installed_os_mount_dir=/target -declare -A c_supported_linux_distributions=([Ubuntu]="18.04 20.04" [UbuntuServer]="18.04 20.04" [LinuxMint]="19.1 19.2 19.3" [Linuxmint]="20 20.1" [elementary]=5.1) +declare -A c_supported_linux_distributions=([Debian]=10 [Ubuntu]="18.04 20.04" [UbuntuServer]="18.04 20.04" [LinuxMint]="19.1 19.2 19.3" [Linuxmint]="20 20.1" [elementary]=5.1) c_temporary_volume_size=12 # gigabytes; large enough - Debian, for example, takes ~8 GiB. c_passphrase_named_pipe=$(dirname "$(mktemp)")/zfs-installer.pp.fifo From 3c6262d882ad208e5d7d5c6bba94b714609db31f Mon Sep 17 00:00:00 2001 From: Saverio Miroddi Date: Mon, 14 Jun 2021 17:32:28 +0200 Subject: [PATCH 84/84] Rename 0.3 backports script It will be kept alongside the standard one. --- install-zfs.sh => install-zfs.backports.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename install-zfs.sh => install-zfs.backports.sh (100%) diff --git a/install-zfs.sh b/install-zfs.backports.sh similarity index 100% rename from install-zfs.sh rename to install-zfs.backports.sh