From fac53badfa3309c3d6e8afb361f04aab4538fe66 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Sun, 1 Aug 2021 18:44:56 +0100 Subject: [PATCH] Create datman-helper-postgres-backup --- .gitignore | 2 +- .../datman_helper_postgres/__init__.py | 0 .../datman_helper_postgres/backup.py | 83 ++++++++++++ .../datman_helper_postgres/restore.py | 0 datman-helper-postgres/setup.py | 119 ++++++++++++++++++ datman/src/commands/backup.rs | 2 +- 6 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 datman-helper-postgres/datman_helper_postgres/__init__.py create mode 100644 datman-helper-postgres/datman_helper_postgres/backup.py create mode 100644 datman-helper-postgres/datman_helper_postgres/restore.py create mode 100644 datman-helper-postgres/setup.py diff --git a/.gitignore b/.gitignore index 5db7f40..bd78447 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,4 @@ /testsuite/yamadatmantestsuite.egg-info __pycache__ - +/datman-helper-postgres/datman_helper_postgres.egg-info diff --git a/datman-helper-postgres/datman_helper_postgres/__init__.py b/datman-helper-postgres/datman_helper_postgres/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/datman-helper-postgres/datman_helper_postgres/backup.py b/datman-helper-postgres/datman_helper_postgres/backup.py new file mode 100644 index 0000000..126f820 --- /dev/null +++ b/datman-helper-postgres/datman_helper_postgres/backup.py @@ -0,0 +1,83 @@ +import json +import os +import subprocess +import sys +from subprocess import PIPE + + +def cli(): + """ + Performs a backup of a Postgres database. + + Parameters: + database: str — the name of the database to back up. + + user: optional str — the name of the Linux user to use to connect to Postgres. + Sudo or SSH will be used to make this happen, if it's specified. + + host: optional str — if specified, the backup will be made using SSH (unless this host is the same as the one + named) + """ + request_info = json.load(sys.stdin) + assert isinstance(request_info, dict) + + database_to_use = request_info["database"] + user_to_use = request_info.get("user") + host_to_use = request_info.get("host") + use_lz4 = request_info.get("use_lz4_for_ssh", True) + + if host_to_use is not None: + hostname = subprocess.check_output("hostname").strip() + if hostname == host_to_use: + host_to_use = None + + command = [] + + if host_to_use is not None: + command.append("ssh") + if user_to_use is not None: + command.append(f"{user_to_use}@{host_to_use}") + else: + command.append(f"{host_to_use}") + elif user_to_use is not None: + command.append("sudo") + command.append("-u") + command.append(user_to_use) + + command.append("pg_dump") + command.append(database_to_use) + + # Where the output of the dump command should go. + output_of_dump = sys.stdout + # The process (if any) that is our LZ4 decompressor. + lz4_process = None + + if use_lz4 and host_to_use is not None: + # Add an LZ4 compressor on the remote side. + command += ["|", "lz4", "--compress", "--stdout"] + + # Then open an LZ4 decompressor on our side. + lz4_process = subprocess.Popen( + ["lz4", "--decompress", "--stdout"], + stdin=subprocess.PIPE, + stdout=sys.stdout, + stderr=sys.stderr, + ) + output_of_dump = lz4_process.stdin + + # we MUST disable shell here otherwise the local side will do both the compression and decompression which would be + # silly! + subprocess.check_call( + command, + stdin=subprocess.DEVNULL, + stdout=output_of_dump, + stderr=sys.stderr, + shell=False, + ) + + if lz4_process is not None: + # must close here, otherwise the decompressor never ends + lz4_process.stdin.close() + exit_code = lz4_process.wait() + if exit_code != 0: + raise ChildProcessError(f"lz4 not happy: {exit_code}") diff --git a/datman-helper-postgres/datman_helper_postgres/restore.py b/datman-helper-postgres/datman_helper_postgres/restore.py new file mode 100644 index 0000000..e69de29 diff --git a/datman-helper-postgres/setup.py b/datman-helper-postgres/setup.py new file mode 100644 index 0000000..62631df --- /dev/null +++ b/datman-helper-postgres/setup.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import io +import os +import sys +from shutil import rmtree + +from setuptools import find_packages, setup, Command + +# Package meta-data. +NAME = "datman_helper_postgres" +DESCRIPTION = "Postgres integration for Datman" +URL = "https://bics.ga/reivilibre/yama" +EMAIL = "reivi@librepush.net" +AUTHOR = "Olivier 'reivilibre'" +REQUIRES_PYTHON = ">=3.7.0" +VERSION = "0.1.0" + +# What packages are required for this module to be executed? +REQUIRED = [] + + +# What packages are optional? +EXTRAS = {} + +# The rest you shouldn't have to touch too much :) +# ------------------------------------------------ +# Except, perhaps the License and Trove Classifiers! +# If you do change the License, remember to change the Trove Classifier for that! + +here = os.path.abspath(os.path.dirname(__file__)) + +# Import the README and use it as the long-description. +# Note: this will only work if 'README.md' is present in your MANIFEST.in file! +try: + with io.open(os.path.join(here, "README.md"), encoding="utf-8") as f: + long_description = "\n" + f.read() +except FileNotFoundError: + long_description = DESCRIPTION + +# Load the package's __version__.py module as a dictionary. +about = {} +if not VERSION: + project_slug = NAME.lower().replace("-", "_").replace(" ", "_") + with open(os.path.join(here, project_slug, "__version__.py")) as f: + exec(f.read(), about) +else: + about["__version__"] = VERSION + + +class UploadCommand(Command): + """Support setup.py upload.""" + + description = "Build and publish the package." + user_options = [] + + @staticmethod + def status(s): + """Prints things in bold.""" + print("\033[1m{0}\033[0m".format(s)) + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + try: + self.status("Removing previous builds…") + rmtree(os.path.join(here, "dist")) + except OSError: + pass + + self.status("Building Source and Wheel (universal) distribution…") + os.system("{0} setup.py sdist bdist_wheel --universal".format(sys.executable)) + + self.status("Uploading the package to PyPI via Twine…") + os.system("twine upload dist/*") + + self.status("Pushing git tags…") + os.system("git tag v{0}".format(about["__version__"])) + os.system("git push --tags") + + sys.exit() + + +# Where the magic happens: +setup( + name=NAME, + version=about["__version__"], + description=DESCRIPTION, + long_description=long_description, + long_description_content_type="text/markdown", + author=AUTHOR, + author_email=EMAIL, + python_requires=REQUIRES_PYTHON, + url=URL, + packages=find_packages(exclude=["tests", "*.tests", "*.tests.*", "tests.*"]), + # If your package is a single module, use this instead of 'packages': + # py_modules=['mypackage'], + entry_points={ + "console_scripts": [ + "datman-helper-postgres-backup=datman_helper_postgres.backup:cli", + "datman-helper-postgres-restore=datman_helper_postgres.restore:cli", + ], + }, + install_requires=REQUIRED, + extras_require=EXTRAS, + include_package_data=True, + # TODO license='GPL3', + classifiers=[ + # Trove classifiers + # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers + "Programming Language :: Python", + "Programming Language :: Python :: 3", + ], +) diff --git a/datman/src/commands/backup.rs b/datman/src/commands/backup.rs index a7e71ec..0e6986e 100644 --- a/datman/src/commands/backup.rs +++ b/datman/src/commands/backup.rs @@ -50,7 +50,7 @@ pub fn open_stdout_backup_process( extra_args: &HashMap, program_name: &str, ) -> anyhow::Result { - let mut child = Command::new(program_name) + let mut child = Command::new(format!("datman-helper-{}-backup", program_name)) .stdout(Stdio::piped()) .stderr(Stdio::inherit()) .stdin(Stdio::piped())