1#!/usr/bin/env python3
2"""
3
4"""
5# mypy: disable-error-code="attr-defined"
6# ruff: noqa: ANN001
7
8# Imports:
9from __future__ import annotations
10
11# ##-- stdlib imports
12import atexit# for @atexit.register
13import collections
14import contextlib
15import datetime
16import enum
17import faulthandler
18import functools as ftz
19import hashlib
20import itertools as itz
21import logging as logmod
22import pathlib as pl
23import re
24import sys
25import time
26import types
27from copy import deepcopy
28from uuid import UUID, uuid1
29from weakref import ref
30
31# ##-- end stdlib imports
32
33# ##-- 3rd party imports
34from jgdv import JGDVError, Mixin, Proto
35from jgdv.logging import JGDVLogConfig
36from jgdv.cli._interface import ParseReport_d
37from jgdv.structs.chainguard import ChainGuard
38from jgdv.structs.dkey import DKey
39from jgdv.structs.locator import JGDVLocator
40from jgdv.structs.metalord.singleton import MLSingleton
41from jgdv.util.plugins.selector import plugin_selector
42from packaging.specifiers import SpecifierSet
43from packaging.version import Version
44
45# ##-- end 3rd party imports
46
47# ##-- 1st party imports
48import doot.errors as DErr # noqa: N812
49from doot import _interface as DootAPI# noqa: N812
50from doot.reporters import BasicReporter
51from doot.reporters._interface import Reporter_p
52
53# ##-- end 1st party imports
54
55# ##-| Local
56from . import _interface as ControlAPI# noqa: N812
57from .loaders._interface import Loader_p
58
59# # End of Imports.
60
61# ##-- types
62# isort: off
63import abc
64import collections.abc
65from collections import defaultdict
66from typing import TYPE_CHECKING, cast, assert_type, assert_never, override
67from typing import Generic, NewType
68# Protocols:
69from typing import Protocol, runtime_checkable
70# Typing Decorators:
71from typing import no_type_check, final, overload
72
73if TYPE_CHECKING:
74 from importlib.metadata import EntryPoint
75 from typing import Final
76 from typing import ClassVar, Any, LiteralString
77 from typing import Never, Self, Literal
78 from typing import TypeGuard
79 from collections.abc import Iterable, Iterator, Callable, Generator
80 from collections.abc import Sequence, Mapping, MutableMapping, Hashable
81
82 from jgdv import Maybe
83 from doot.errors import DootError
84
85 type Logger = logmod.Logger
86 type Loadable = DootAPI.Loadable
87
88##--|
89
90# isort: on
91# ##-- end types
92
93##-- logging
94logging = logmod.getLogger(__name__)
95##-- end logging
96
97# Vars:
98##--| Controllers
99
[docs]
100class StartupController:
101 type DO = DootOverlord
102
[docs]
103 def null_setup(self, obj:DO) -> None:
104 """
105 Doesn't load anything but constants,
106 Used for initialising Doot when testing.
107 Doesn't set the is_setup flag.
108 """
109 if obj.is_setup:
110 return
111
112 self._load_constants(obj, target=DootAPI.constants_file)
113 self._load_aliases(obj, target=DootAPI.aliases_file)
114
[docs]
115 def setup(self, obj:DO, *, targets:Maybe[list[Loadable]]=None, prefix:Maybe[str]=None) -> None:
116 """
117 The core requirement to call before any other doot code is run.
118 loads the config files, so everything else can retrieve values when imported.
119
120 `prefix` removes a prefix from the loaded data.
121 eg: 'tool.doot' for if putting doot settings in a pyproject.toml
122
123 targets=False is for loading nothing, for testing
124 """
125 if obj.is_setup:
126 obj.report.gen.user("doot.setup called even though doot is already set up")
127
128 self._load_config(obj, targets=targets, prefix=prefix or DootAPI.TOOL_PREFIX)
129 self._load_constants(obj,
130 target=obj.config.on_fail(None).startup.constants_file(wrapper=pl.Path))
131 self._load_aliases(obj,
132 target=obj.config.on_fail(None).startup.aliases_file(wrapper=pl.Path),
133 force=True)
134 self._load_locations(obj)
135
136 # add global task state as a DKey expansion source
137 DKey.add_sources(obj.global_task_state)
138 obj.is_setup = True
139
[docs]
140 def _load_config(self, obj:DO, *, targets:Maybe[list[Loadable]], prefix:Maybe[str]) -> None: # noqa: PLR0912
141 """ Load a specified config, or one of the defaults if it exists """
142 x : Any
143 target_paths : list[pl.Path]
144 existing_targets : list[pl.Path]
145 ##--|
146 match targets:
147 case list() if bool(targets) and all([isinstance(x, pl.Path) for x in targets]):
148 target_paths = [pl.Path(x) for x in targets] # type: ignore[arg-type]
149 case list() if bool(targets):
150 raise TypeError("Doot Config Targets should be pathlib.Path's", targets)
151 case None | []:
152 target_paths = [pl.Path(x) for x in obj.constants.paths.DEFAULT_LOAD_TARGETS]
153
154 logging.log(0, "Loading Doot Config, version: %s targets: %s", DootAPI.__version__, target_paths)
155 assert(isinstance(target_paths, list))
156 match [x for x in target_paths if x.is_file()]:
157 case [] if bool(target_paths):
158 raise DErr.MissingConfigError("No Doot data found")
159 case []:
160 existing_targets = []
161 case [*xs]:
162 existing_targets = xs
163 case x:
164 raise TypeError(type(x))
165
166 ##--| Load config Files
167 for existing in existing_targets:
168 try:
169 config = ChainGuard.load(existing)
170 except OSError as err:
171 raise DErr.InvalidConfigError(existing_targets, *err.args) from err
172 else:
173 match existing:
174 case x if x.name == DootAPI.PYPROJ_TOML and DootAPI.TOOL_PREFIX not in config:
175 logging.debug("Pyproject has no doot config, ignoring")
176 continue
177 case x if x.name == DootAPI.DOOT_TOML:
178 chopped = config
179 case x:
180 chopped = config.remove_prefix(prefix)
181 if not bool(chopped):
182 continue
183
184 conf_ver = chopped.on_fail(None).startup.doot_version()
185 obj.verify_config_version(conf_ver, source=existing)
186 obj.config = ChainGuard.merge(chopped, obj.config._table()) # type: ignore[arg-type]
187 obj.configs_loaded_from.append(existing)
188 obj.update_global_task_state(obj.config, source=str(existing_targets))
189
[docs]
190 def _load_constants(self, obj:DO, *, target:Maybe[Loadable]=None) -> None:
191 """ Load the override constants if the loaded base config specifies one
192 Modifies the global `doot.constants`
193 """
194 match target:
195 case None:
196 pass
197 case pl.Path() as const_file if const_file.exists():
198 obj.report.gen.trace("Loading Constants")
199 base_data = ChainGuard.load(const_file)
200 obj.verify_config_version(base_data.on_fail(None).doot_version(), source=const_file)
201 obj.constants = base_data.remove_prefix(DootAPI.CONSTANT_PREFIX)
202
[docs]
203 def _load_aliases(self, obj:DO, *, target:Maybe[Loadable]=None, force:bool=False) -> None:
204 """ Load plugin aliases from a toml file
205
206 if forced, will append additional aliases on top of existing
207 """
208 final_aliases : dict = defaultdict(dict)
209 base_data : dict|ChainGuard = {}
210 target = target or DootAPI.aliases_file
211
212 match bool(obj.aliases), force:
213 case False, _:
214 pass
215 case True, True:
216 final_aliases.update(dict(obj.aliases._table()))
217 case True, _:
218 raise RuntimeError("Tried to re-initialise aliases")
219
220 obj.report.gen.trace("Initalising Aliases")
221 match target:
222 case pl.Path() as source if source.exists():
223 obj.report.gen.trace("Loading Aliases: %s", source) # type: ignore[arg-type]
224 base_data = ChainGuard.load(source)
225 assert(isinstance(base_data, ChainGuard))
226 obj.verify_config_version(base_data.on_fail(None).doot_version(), source=source)
227 base_data = base_data.remove_prefix(DootAPI.ALIAS_PREFIX)
228 case pl.Path() as source:
229 obj.report.gen.user("Alias File Not Found: %s", source)
230 case x:
231 raise TypeError(type(x))
232
233 ##--|
234 # Flatten the lists
235 for key,_val in base_data.items():
236 _val = cast("list[dict]", _val)
237 final_aliases[key] = {k:v for x in _val for k,v in x.items()}
238 else:
239 obj.aliases = ChainGuard(final_aliases)
240
[docs]
241 def _load_locations(self, obj:DO) -> None:
242 """ Load and update the JGDVLocator db
243 """
244 obj.report.gen.trace("Loading Locations")
245 # Load Initial locations
246 for loc in obj.config.on_fail([]).locations():
247 try:
248 for name in loc.keys():
249 obj.report.gen.trace("+ %s", name)
250 obj.locs.update(loc, strict=False)
251 except (JGDVError, ValueError) as err:
252 obj.report.gen.error("Location Loading Failed: %s (%s)", loc, err)
253
[docs]
254class PluginsController:
255 type DO = DootOverlord
256
[docs]
257 def load(self, obj:DO) -> None:
258 self._load_plugins(obj)
259 self._load_commands(obj, loader=obj.config.on_fail("default").startup.loaders.command())
260 self._load_tasks(obj, loader=obj.config.on_fail("default").startup.loaders.task())
261
[docs]
262 def _load_plugins(self, obj:DootOverlord) -> None:
263 """ Use the plugin loader to find all applicable `importlib.EntryPoint`s """
264 # ##-- 1st party imports
265 from doot.control.loaders.plugin import PluginLoader # noqa: PLC0415
266
267 # ##-- end 1st party imports
268 try:
269 plugin_loader = PluginLoader()
270 plugin_loader.setup()
271 obj.loaded_plugins = plugin_loader.load()
272 obj.update_aliases(data=obj.loaded_plugins) # type: ignore[attr-defined]
273 except DErr.PluginError as err:
274 obj.report.gen.error("Plugins Not Loaded Due to Error: %s", err) # type: ignore[attr-defined]
275 raise
276
[docs]
277 def _load_commands(self, obj:DootOverlord, *, loader:str="default") -> None:
278 """ Select Commands from the discovered loaded_plugins,
279 using the preferred cmd loader or the default
280 """
281 if not bool(obj.loaded_plugins):
282 raise RuntimeError("Tried to Load Commands without having loaded Plugins")
283
284 match plugin_selector(obj.loaded_plugins.on_fail([], list).command_loader(),
285 target=loader):
286 case type() as ctor:
287 cmd_loader = ctor()
288 case x:
289 raise TypeError(type(x))
290
291 match cmd_loader:
292 case Loader_p():
293 try:
294 cmd_loader.setup(obj.loaded_plugins)
295 obj.loaded_cmds = cmd_loader.load()
296 except DErr.PluginError as err:
297 obj.report.gen.error("Commands Not Loaded due to Error: %s", err) # type: ignore[attr-defined]
298 obj.loaded_cmds = ChainGuard()
299 case x:
300 raise TypeError("Unrecognized loader type", x)
301
[docs]
302 def _load_tasks(self, obj:DootOverlord, *, loader:str="default") -> None:
303 """ Load task entry points, using the preferred task loader,
304 or the default
305 """
306 x : Any
307 task_loader : Loader_p
308 ##--|
309 match plugin_selector(obj.loaded_plugins.on_fail([], list).task_loader(),
310 target=loader):
311 case type() as ctor:
312 task_loader = ctor()
313 case x:
314 raise TypeError(type(x))
315
316 match task_loader:
317 case Loader_p():
318 task_loader.setup(obj.loaded_plugins)
319 obj.loaded_tasks = task_loader.load()
320 case x:
321 raise TypeError("Unrecognised loader type", x)
322
323##--| Overlord
324
[docs]
325@Proto(ControlAPI.Overlord_i)
326class DootOverlord:
327 """
328 The main control point of Doot
329 The setup logic of doot.
330
331 As Doot uses loaded config data throughout, using the doot.config.on_fail... pattern,
332 the top-level package 'doot', uses a module getattr to offload attribute access to this class.
333
334 Adapted from https://stackoverflow.com/questions/880530
335 """
336
337 _startup : ClassVar[StartupController] = StartupController()
338 _plugin : ClassVar[PluginsController] = PluginsController()
339
340 __version__ : str
341 global_task_state : dict[str, Any]
342 path_ext : list[str]
343 configs_loaded_from : list[str|pl.Path]
344 is_setup : bool
345
346 def __init__(self, *args:Any, **kwargs:Any):
347 super().__init__(*args, **kwargs)
348 logging.info("Creating Overlord")
349 empty_chain = ChainGuard()
350 self.__version__ = DootAPI.__version__
351 self.global_task_state = {}
352 self.path_ext = []
353 self.configs_loaded_from = []
354 self.is_setup = False
355 self.config = empty_chain
356 self.constants = empty_chain
357 self.aliases = empty_chain
358 self.loaded_plugins = empty_chain
359 self.loaded_cmds = empty_chain
360 self.loaded_tasks = empty_chain
361 # TODO Remove this:
362 self.cmd_aliases = ChainGuard()
363 self.args = ChainGuard() # parsed arg access
364 self.locs = JGDVLocator(pl.Path.cwd())
365 self.report = BasicReporter()
366
367 DootOverlord._startup.null_setup(self)
368
369 @property
370 def report(self) -> Reporter_p:
371 return self._reporter
372
[docs]
373 @report.setter
374 def report(self, rep:Any) -> None:
375 self._set_reporter(rep)
376
377 ##--| internal methods
378
[docs]
379 def _set_reporter(self, rep:Any) -> None:
380 match rep:
381 case Reporter_p():
382 self._reporter = rep
383 case x:
384 raise TypeError(type(x))
385
386 ##--| public methods
387
[docs]
388 def setup(self, *, targets:Maybe[list[Loadable]]=None, prefix:Maybe[str]=None) -> None:
389 self._startup.setup(self, targets=targets, prefix=prefix)
390 self.update_import_path()
391
[docs]
392 def load(self) -> None:
393 self._plugin.load(self)
394
[docs]
395 def load_reporter(self, target:str="default") -> None:
396 if not bool(self.loaded_plugins):
397 raise RuntimeError("Tried to Load Reporter without loading loaded_plugins")
398
399 match plugin_selector(self.loaded_plugins.on_fail([], list).reporter(),
400 target=target):
401 case type() as ctor:
402 self.report = ctor() # type: ignore[attr-defined]
403 case x:
404 raise TypeError(type(x))
405
[docs]
406 def verify_config_version(self, ver:Maybe[str], source:Maybe[str|pl.Path], *, override:Maybe[str]=None) -> None:
407 """Ensure the config file is compatible with doot
408
409 Compatibility is based on MAJOR.MINOR and discards PATCH
410
411 Raises a VersionMismatchError otherwise if they aren't compatible
412 """
413 doot_ver = Version(override or DootAPI.__version__)
414 test_ver = SpecifierSet(f"~={doot_ver.major}.{doot_ver.minor}.0")
415 match ver:
416 case str() as x if x in test_ver:
417 return
418 case str() as x:
419 raise DErr.VersionMismatchError("Config File is incompatible with this version of doot (%s, %s) : %s : %s", DootAPI.__version__, test_ver, x, source)
420 case _:
421 raise DErr.VersionMismatchError("No Doot Version Found in config file: %s", source)
422
[docs]
423 def update_aliases(self, *, data:dict|ChainGuard) -> None:
424 """
425 Update aliases with a dict-like of loaded mappings
426 """
427 final_aliases : dict
428 if not bool(data):
429 return
430
431 final_aliases = defaultdict(dict)
432 final_aliases.update(dict(self.aliases._table()))
433
434 for key,eps in data.items():
435 eps = cast("list[EntryPoint]", eps)
436 update = {x.name:x.value for x in eps} # type: ignore[union-attr]
437 final_aliases[key].update(update)
438 else:
439 self.aliases = ChainGuard(final_aliases)
440
[docs]
441 def update_global_task_state(self, data:ChainGuard, *, source:Maybe[str]=None) -> None:
442 """ Try to Update the shared global state.
443 Will try to get data[doot._interface.GLOBAL_STATE_KEY] data and add it to the global task state
444
445 toml in [[state]] segments is merged here
446 """
447 if source is None:
448 raise ValueError("Updating Global Task State must have a source")
449
450 self.report.gen.detail("Updating Global State from: %s", source)
451 if not isinstance(data, dict|ChainGuard):
452 raise DErr.GlobalStateMismatch("Not a dict", data)
453
454 match data.on_fail([])[DootAPI.GLOBAL_STATE_KEY]():
455 case []:
456 return
457 case [*xs]:
458 updates = xs
459 case dict() as x:
460 updates = [x]
461 case x:
462 raise TypeError(type(x))
463
464 for up in updates:
465 for x,y in up.items():
466 if x not in self.global_task_state:
467 self.global_task_state[x] = y
468 elif self.global_task_state[x] != y:
469 raise DErr.GlobalStateMismatch(x, y, source)
470
[docs]
471 def update_import_path(self, *paths:pl.Path) -> None:
472 """ Add locations to the python path for task local code importing
473 Modifies the global `sys.path`
474 """
475 x : Any
476 combined : set[pl.Path]
477 ##--| Wrappers
478 def loc_wrapper(x) -> list[pl.Path]:
479 return [self.locs[y] for y in x]
480
481 ##--|
482 self.report.gen.trace("Updating Import Path")
483 match paths:
484 case None | []:
485 task_sources = self.config.on_fail([self.locs[".tasks"]], list).startup.sources.tasks(wrapper=loc_wrapper)
486 task_code = self.config.on_fail([self.locs[".tasks"]], list).startup.sources.code(wrapper=loc_wrapper)
487 combined = set(task_sources + task_code)
488 case [*xs]:
489 combined = set(paths)
490
491 assert(isinstance(combined, set))
492 for source in combined:
493 match source:
494 case pl.Path() as x if not x.exists():
495 continue
496 case pl.Path() as x if not x.is_dir():
497 continue
498 case pl.Path() as x:
499 # sys.path does not play nice with pl.Path
500 source_str = str(source.expanduser().resolve())
501 case x:
502 raise TypeError("Bad Type for adding to sys.path", x)
503
504 match source_str:
505 case x if x in sys.path:
506 continue
507 case str():
508 self.report.gen.trace("sys.path += %s", str(source))
509 sys.path.append(source_str)
510 else:
511 self.report.gen.trace("Import Path Updated")
512
[docs]
513 def update_cmd_args(self, data:ParseReport_d|dict, *, override:bool=False) -> None:
514 """ update global args that cmd's use for control flow """
515 prepared : dict
516 ##--|
517 match data:
518 case _ if bool(self.args) and not override:
519 raise ValueError("Setting Parsed args but its already set")
520 case {"prog": dict() as prog, "cmds": dict() as cmds, "subs": list() as subs, "help": bool() as _help}:
521 prepared = {
522 "prog" : prog,
523 "cmds" : cmds,
524 "subs" : subs,
525 "help" : _help,
526 }
527 case ParseReport_d():
528 prepared = data.to_dict()
529 case x:
530 raise TypeError(type(x))
531
532 ##--|
533 assert(bool(prepared))
534 assert(all(x in prepared for x in ["prog", "cmds", "subs", "help"]))
535 assert(isinstance(prepared['cmds'], dict))
536 assert(isinstance(prepared['subs'], dict))
537 # Handle 'help'
538 match prepared:
539 case _ if not prepared['help']:
540 pass
541 case {"subs": _subs} if bool(_subs):
542 prepared['cmds'].clear()
543 prepared['cmds']['help'] = []
544 prepared['cmds']['help'] += [{"name":"help", "args":{"target": x}} for x in _subs.keys()]
545 case {"cmds": _cmds} if bool(_cmds):
546 keys = list(_cmds.keys())
547 prepared['cmds'].clear()
548 prepared['cmds']['help'] = []
549 prepared['cmds']['help'] += [{"name":"help", "args":{"target": x}} for x in keys]
550
551
552 self.args = ChainGuard(prepared)
553
554##--| Facade
555
[docs]
556class OverlordFacade(types.ModuleType):
557 """
558 A Facade for the overlord, to be used as the module class
559 of the root package 'doot'.
560
561 """
562 _overlord : ControlAPI.Overlord_i
563
564 def __init__(self, *args, **kwargs) -> None: # noqa: ANN002, ANN003
565 super().__init__(*args, **kwargs)
566 self._overlord = cast("ControlAPI.Overlord_i", DootOverlord())
567
568 @override
569 def __getattr__(self, key):
570 return getattr(self._overlord, key)