reivilibre/scone
1
0
Fork 0
Code Issues 21 Pull Requests Releases Wiki Activity

Variables and templating (#5)

Browse Source 
* Fix issue with using variables in for loops

* Fix critical bug: mutable default value in attrs class

* Substitute in vars in recipe arguments

* Mark failed recipes on the dot output

* Add support for Jinja2 templating

* Antilint
This commit is contained in:
reivilibre 2020-10-23 09:32:06 +01:00 committed by GitHub
parent c970fe6ca5
commit 04241dc47f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 115 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
scone/default/steps/fridge_steps.py
View File

@ -2,7 +2,7 @@ from enum import Enum
from pathlib import Path, PurePath
from typing import List, Optional, Tuple, Union
from jinja2 import Template
from jinja2 import DictLoader, Environment
from scone.head.head import Head
from scone.head.kitchen import Kitchen
@ -77,10 +77,24 @@ async def load_and_transform(
data = head.secret_access.decrypt_bytes(data)
elif meta == FridgeMetadata.TEMPLATE:
# pass through Jinja2
template = Template(data.decode())
proxies = kitchen.get_dependency_tracker().get_j2_compatible_dep_var_proxies(
head.variables[sous]
)
data = template.render(proxies).encode()
try:
env = Environment(
loader=DictLoader({str(fullpath): data.decode()}), autoescape=False
)
template = env.get_template(str(fullpath))
proxies = kitchen.get_dependency_tracker().get_j2_var_proxies(
head.variables[sous]
)
data = template.render(proxies).encode()
except Exception as e:
raise RuntimeError(f"Error templating: {fullpath}") from e
# try:
# return jinja2.utils.concat(
# template.root_render_func(template.new_context(proxies))
# )
# except Exception:
# template.environment.handle_exception()
print("data", fullpath, data)
return data

3
scone/head/dag.py
View File

@ -30,6 +30,9 @@ class RecipeState(Enum):
# This recipe has not been cooked because it didn't need to be.
SKIPPED = 10
# This recipe failed.
FAILED = -1
@staticmethod
def is_completed(state):
return state in (RecipeState.COOKED, RecipeState.SKIPPED)

60
scone/head/dependency_tracking.py
View File

@ -1,6 +1,7 @@
import json
import logging
import time
from copy import deepcopy
from hashlib import sha256
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
@ -12,6 +13,7 @@ from aiosqlite import Connection
from scone.head.dag import Resource
from scone.head.recipe import recipe_name_getter
from scone.head.variables import Variables
if TYPE_CHECKING:
from scone.head.dag import RecipeDag
@ -118,7 +120,8 @@ class DependencyTracker:
def register_variable(self, variable: str, value: Union[dict, str, int]):
# self._vars[variable] = value
raise NotImplementedError("time")
# TODO(implement)
logger.critical("not implemented: register var %s", variable)
def register_fridge_file(self, desugared_path: str):
# TODO this is not complete
@ -131,19 +134,48 @@ class DependencyTracker:
file_res = Resource("file", path, sous=sous)
self.watch(file_res)
# def get_j2_compatible_dep_var_proxies(
# self, variables: Variables
# ) -> Dict[str, "DependencyVarProxy"]:
# # XXX BROKEN does not work for overrides
# result = {}
#
# if len("1"):
# raise NotImplementedError("BROKEN")
#
# for key, vars in variables.toplevel().items():
# result[key] = DependencyVarProxy(self, vars, key + ".")
#
# return result
def get_j2_var_proxies(
self, variables: Variables
) -> Dict[str, "DependencyVarProxy"]:
result = {}
for key in variables.toplevel():
result[key] = DependencyVarProxy(key, variables, self)
return result
class DependencyVarProxy:
def __init__(
self,
current_path_prefix: Optional[str],
vars: Variables,
tracker: DependencyTracker,
):
self._current_path_prefix = current_path_prefix
self._vars = vars
self._tracker = tracker
def raw_(self) -> Dict[str, Any]:
if not self._current_path_prefix:
raw_dict = self._vars.toplevel()
else:
raw_dict = self._vars.get_dotted(self._current_path_prefix)
self._tracker.register_variable(self._current_path_prefix or "", raw_dict)
return deepcopy(raw_dict)
def __getattr__(self, name: str) -> Union["DependencyVarProxy", Any]:
if not self._current_path_prefix:
dotted_path = name
else:
dotted_path = f"{self._current_path_prefix}.{name}"
raw_value = self._vars.get_dotted(dotted_path)
if isinstance(raw_value, dict):
return DependencyVarProxy(dotted_path, self._vars, self._tracker)
else:
self._tracker.register_variable(dotted_path, raw_value)
return raw_value
class DependencyCache:

19
scone/head/dot_emitter.py
View File

@ -5,13 +5,14 @@ from scone.head.dag import RecipeDag, RecipeState, Resource, Vertex
from scone.head.recipe import Recipe, recipe_name_getter
state_to_colour = {
RecipeState.LOADED: "#000000",
RecipeState.PREPARED: "azure",
RecipeState.PENDING: "pink",
RecipeState.COOKABLE: "gold",
RecipeState.COOKED: "darkolivegreen1",
RecipeState.SKIPPED: "cadetblue1",
RecipeState.BEING_COOKED: "darkorange1",
RecipeState.LOADED: ("white", "black"),
RecipeState.PREPARED: ("azure", "black"),
RecipeState.PENDING: ("pink", "black"),
RecipeState.COOKABLE: ("gold", "black"),
RecipeState.COOKED: ("darkolivegreen1", "black"),
RecipeState.SKIPPED: ("cadetblue1", "black"),
RecipeState.BEING_COOKED: ("darkorange1", "black"),
RecipeState.FAILED: ("black", "orange"),
}
@ -32,10 +33,10 @@ def emit_dot(dag: RecipeDag, path_out: Path) -> None:
f"{recipe_name_getter(vertex.__class__)}"
f" [{rec_meta.incoming_uncompleted}]"
)
colour = state_to_colour[rec_meta.state]
colour, text_colour = state_to_colour[rec_meta.state]
fout.write(
f'\t{vertex_id} [shape=box, label="{label}",'
f" style=filled, fillcolor={colour}];\n"
f" style=filled, fontcolor={text_colour}, fillcolor={colour}];\n"
)
elif isinstance(vertex, Resource):
label = str(vertex).replace("\\", "\\\\").replace('"', '\\"')

6
scone/head/kitchen.py
View File

@ -208,7 +208,11 @@ class Kitchen:
self._dependency_trackers[next_job] = DependencyTracker(
DependencyBook(), dag, next_job
)
await next_job.cook(self)
try:
await next_job.cook(self)
except Exception as e:
meta.state = RecipeState.FAILED
raise RuntimeError(f"Recipe {next_job} failed!") from e
eprint(f"cooked {next_job}")
# TODO cook
# TODO store depbook

28
scone/head/menu_reader.py
View File

@ -10,11 +10,11 @@ import textx
from scone.head.dag import RecipeDag, Resource
from scone.head.recipe import RecipeContext
from scone.head.variables import Variables
if typing.TYPE_CHECKING:
from scone.head.head import Head
from scone.head.recipe import Recipe
from scone.head.variables import Variables
def _load_grammar():
@ -69,10 +69,10 @@ class MenuBlock:
user_directive: Optional[str] = None
sous_directive: Optional[str] = None
for_directives: List[ForDirective] = []
import_directives: List[str] = []
recipe_edges: List[RecipeEdgeDirective] = []
resource_edges: List[ResourceEdgeDirective] = []
for_directives: List[ForDirective] = attr.ib(factory=list)
import_directives: List[str] = attr.ib(factory=list)
recipe_edges: List[RecipeEdgeDirective] = attr.ib(factory=list)
resource_edges: List[ResourceEdgeDirective] = attr.ib(factory=list)
@attr.s(auto_attribs=True, eq=False)
@ -89,9 +89,9 @@ class MenuRecipe:
user_directive: Optional[str] = None
sous_directive: Optional[str] = None
for_directives: List[ForDirective] = []
recipe_edges: List[RecipeEdgeDirective] = []
resource_edges: List[ResourceEdgeDirective] = []
for_directives: List[ForDirective] = attr.ib(factory=list)
recipe_edges: List[RecipeEdgeDirective] = attr.ib(factory=list)
resource_edges: List[ResourceEdgeDirective] = attr.ib(factory=list)
def convert_textx_value(txvalue) -> Any:
@ -336,11 +336,13 @@ class MenuLoader:
hierarchical_source=hierarchical_source, # XXX
human=recipe.human,
)
args = recipe.arguments # noqa
# XXX sub in vars
instance: Recipe = recipe_class.new(
context, recipe.arguments, self._head
)
try:
args = _vars.substitute_in_dict_copy(recipe.arguments)
except KeyError as ke:
raise KeyError(
f"When substituting for {hierarchical_source} / {recipe}"
) from ke
instance: Recipe = recipe_class.new(context, args, self._head)
self._recipes[recipe][(sous, for_indices)] = instance
self._dag.add(instance)

20
scone/head/variables.py
View File

@ -1,5 +1,6 @@
from copy import deepcopy
from enum import Enum
from typing import Any, Dict, List, NamedTuple, Optional
from typing import Any, Dict, List, NamedTuple, Optional, Set
ExpressionPart = NamedTuple("ExpressionPart", [("kind", str), ("value", str)])
@ -131,14 +132,14 @@ class Variables:
self.set_dotted(var_name, sub_val)
return sub_val
else:
raise KeyError(f"No variable '{incoming}'")
raise KeyError(f"No variable '{var_name}'")
out = ""
for part in parsed:
if part.kind == "literal":
out += part.value
elif part.kind == "variable":
var_name = parsed[0].value
var_name = part.value
if self.has_dotted(var_name):
out += str(self.get_dotted(var_name))
elif var_name in incoming:
@ -147,7 +148,7 @@ class Variables:
self.set_dotted(var_name, sub_val)
out += str(sub_val)
else:
raise KeyError(f"No variable '{incoming}'")
raise KeyError(f"No variable '{var_name}'")
return out
def load_vars_with_substitutions(self, incoming: Dict[str, Any]):
@ -170,5 +171,16 @@ class Variables:
elif isinstance(v, str):
dictionary[k] = self.eval(v)
def substitute_in_dict_copy(self, dictionary: Dict[str, Any]):
new_dict = deepcopy(dictionary)
self.substitute_inplace_in_dict(new_dict)
return new_dict
def toplevel(self):
return self._vars
def keys(self) -> Set[str]:
keys = set(self._vars.keys())
if self._delegate:
keys.update(self._delegate.keys())
return keys