From 6e16ac7ec6b017758e79941042b39354c621747e Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 16 Mar 2022 12:54:42 +0000 Subject: [PATCH] Add head-only recipes, plus start one to do Hurricane DNS --- scone/default/recipes/dns_he.py | 126 ++++++++++++++++++++++++++++++++ scone/head/cli/__init__.py | 1 + scone/head/head.py | 11 ++- scone/head/kitchen.py | 5 +- scone/head/recipe.py | 17 +++++ setup.py | 8 +- 6 files changed, 165 insertions(+), 3 deletions(-) create mode 100644 scone/default/recipes/dns_he.py diff --git a/scone/default/recipes/dns_he.py b/scone/default/recipes/dns_he.py new file mode 100644 index 0000000..b446fc1 --- /dev/null +++ b/scone/default/recipes/dns_he.py @@ -0,0 +1,126 @@ +# Copyright 2022, Olivier 'reivilibre'. +# +# This file is part of Scone. +# +# Scone is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Scone is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Scone. If not, see . +from asyncio import Lock +from typing import Any, Dict, List, Optional, Tuple, Union + +import attr +from HurricaneDNS import HurricaneDNS + +from scone.head.head import Head +from scone.head.kitchen import Kitchen, Preparation +from scone.head.recipe import HeadRecipe, RecipeContext +from scone.head.utils import check_type + + +@attr.s(auto_attribs=True) +class DnsRecord: + pass + + +def parse_records(given: Union[Any, Dict[str, Any]]) -> List[DnsRecord]: + pass + + +@attr.s(auto_attribs=True) +class HeRecord: + id: str + status: Optional[str] + host: str + type: str + ttl: str + # MX Priority + mx: str + value: str + extended: str + + +@attr.s(auto_attribs=True) +class HeDomain: + domain: str + id: str + type: str + records: Optional[List[HeRecord]] + + +@attr.s(auto_attribs=True) +class HurricaneElectricCache: + client: HurricaneDNS + lock: Lock + domains: Dict[str, HeDomain] + + +# Tuple from (id(head), user, password) → HE DNS cache) +# If Scone is ever long-living, this could leak, but it doesn't so it won't matter +HE_CLIENT_CACHE: Dict[Tuple[int, str, str], HurricaneElectricCache] = {} +CLIENT_CACHE_LOCK: Lock = Lock() + + +class HurricaneElectricDns(HeadRecipe): + _NAME = "dns-hurricane" + + def __init__(self, recipe_context: RecipeContext, args: dict, head: Head): + super().__init__(recipe_context, args, head) + + self.username = check_type(args.get("username"), str) + self.password = check_type(args.get("password"), str) + self.domain = check_type(args.get("domain"), str) + + self.records = parse_records(args.get("records")) + + def prepare(self, preparation: Preparation, head: Head) -> None: + super().prepare(preparation, head) + + async def _get_client(self, head: Head) -> HurricaneElectricCache: + async with CLIENT_CACHE_LOCK: + cache_key = id(head), self.username, self.password + if cache_key in HE_CLIENT_CACHE: + # Happy days + return HE_CLIENT_CACHE[cache_key] + + # TODO(performance): this takes about 3 sec; move it to an executor thread + client = HurricaneDNS(self.username, self.password) + + domains = {} + for domain in client.list_domains(): + dom = HeDomain(**domain) + domains[dom.domain] = dom + + entry = HurricaneElectricCache(client, Lock(), domains) + HE_CLIENT_CACHE[cache_key] = entry + + return entry + + async def _get_records( + self, cached: HurricaneElectricCache, domain: HeDomain + ) -> List[HeRecord]: + async with cached.lock: + if domain.records is not None: + return domain.records + + domain.records = [ + HeRecord(**row) for row in cached.client.list_records(domain.domain) + ] + return domain.records + + async def cook(self, kitchen: Kitchen) -> None: + kitchen.get_dependency_tracker().ignore() + cached = await self._get_client(kitchen.head) + + records = await self._get_records(cached, cached.domains[self.domain]) + + for record in records: + print(record) diff --git a/scone/head/cli/__init__.py b/scone/head/cli/__init__.py index b58f230..107cab2 100644 --- a/scone/head/cli/__init__.py +++ b/scone/head/cli/__init__.py @@ -92,6 +92,7 @@ async def cli_async() -> int: eprint(f"Selected the following souss: {', '.join(hosts)}") + # Load variables for the head as well. head.load_variables(hosts) head.load_menus(menu_subset, hosts) diff --git a/scone/head/head.py b/scone/head/head.py index 8cb8ade..8397d30 100644 --- a/scone/head/head.py +++ b/scone/head/head.py @@ -37,6 +37,8 @@ from scone.head.variables import Variables, merge_right_into_left_inplace logger = logging.getLogger(__name__) +SPECIAL_HEAD_SOUS = "head" + class Head: def __init__( @@ -52,6 +54,8 @@ class Head: self.recipe_loader = recipe_loader self.dag = RecipeDag() self.souss = sous + # Special override: head sous. + self.souss[SPECIAL_HEAD_SOUS] = {"user": ""} self.groups = groups self.secret_access = secret_access self.variables: Dict[str, Variables] = dict() @@ -79,7 +83,8 @@ class Head: sous = head_data.get("sous", dict()) groups = head_data.get("group", dict()) - groups["all"] = list(sous.keys()) + groups["all_plus_head"] = list(sous.keys()) + groups["all"] = list(sous.keys() - "head") pools = Pools() @@ -91,6 +96,10 @@ class Head: out_chilled: Dict[str, Any] = {} vardir = Path(self.directory, "vars", who_for) + # TODO(feature): is this needed? + # if not vardir.exists(): + # return out_chilled, out_frozen + logger.debug("preloading vars for %s in %s", who_for, str(vardir)) for file in vardir.glob("*.vf.toml"): diff --git a/scone/head/kitchen.py b/scone/head/kitchen.py index 74e491f..327c7d3 100644 --- a/scone/head/kitchen.py +++ b/scone/head/kitchen.py @@ -40,7 +40,7 @@ from scone.head.dependency_tracking import ( DependencyTracker, hash_dict, ) -from scone.head.head import Head +from scone.head.head import SPECIAL_HEAD_SOUS, Head from scone.head.recipe import Recipe from scone.sous import utensil_namer from scone.sous.utensils import Utensil @@ -167,6 +167,9 @@ class Kitchen: return ChanProHead(cp, root) + if host == SPECIAL_HEAD_SOUS: + raise ValueError("Can't connect to special 'head' sous over SSH") + hostuser = (host, user) if hostuser not in self._chanproheads: self._chanproheads[hostuser] = asyncio.create_task(new_conn()) diff --git a/scone/head/recipe.py b/scone/head/recipe.py index 6c4592e..9b22c53 100644 --- a/scone/head/recipe.py +++ b/scone/head/recipe.py @@ -81,3 +81,20 @@ class Recipe: f"{cls.__name__} {self.recipe_context.human}" f" on {self.recipe_context.sous} ({self.arguments})" ) + + +class HeadRecipe(Recipe): + def __init__( + self, recipe_context: RecipeContext, args: Dict[str, Any], head: "Head" + ): + super().__init__(recipe_context, args, head) + if recipe_context.sous != "head": + myname = self.__class__._NAME + raise ValueError( + f"[[{myname}]] is a head recipe, so should be run with @sous = head" + ) + + def prepare(self, preparation: "Preparation", head: "Head") -> None: + # Don't add a requirement for an os-user, since it's a head recipe + # that doesn't have one. + pass diff --git a/setup.py b/setup.py index 7aefb13..4fa98ca 100644 --- a/setup.py +++ b/setup.py @@ -54,12 +54,18 @@ EX_HEAD = [ "typeguard", "textx", ] +EX_HEAD_HEDNS = [ + "hurricanedns~=1.0.2", +] +EX_HEAD_ALL = EX_HEAD + EX_HEAD_HEDNS EX_DEV = EX_SOUS_ALL + EX_HEAD + EX_DEV_MYPY # What packages are optional? EXTRAS = { - "head": EX_HEAD, + "head-core": EX_HEAD, + "head": EX_HEAD_ALL, + "head-hedns": EX_HEAD_HEDNS, "sous": EX_SOUS_ALL, "sous-core": EX_SOUS_BASE, "sous-pg": EX_SOUS_PG,