#!/usr/bin/env python3 """ Define a NbCourse object containing: - class methods for building html pages and pdf book - Doit tasks defined in task_*() functions """ from pathlib import Path, PosixPath import yaml from doit.tools import create_folder, config_changed from .utils import update_material, clean_tree, get_file_list, update_dict, \ zip_files from .pages import HomePage, MarkdownPage from .book import Book import nbformat from nbconvert.preprocessors import ExecutePreprocessor from nbconvert import HTMLExporter, SlidesExporter from .mydoit import MyDoitMain, ClassTaskLoader import logging import sys log = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) DEFAULT_CONFIG_FILE = "nbcourse.yml" class NbCourse: default_conf = { 'theme': { 'dir': 'theme/default', 'material': ('css', 'img') }, 'nb': { 'dir': '.', 'timeout': 60, 'material': () }, 'pages': { 'dir': 'pages', 'home': 'index.html' }, 'book': { 'titlepage': 'titlepage.tex', 'file': None, }, 'local_reveal': False, 'slug_title': 'course', 'output_dir': 'build', 'title': 'A course', 'subtitle': None, 'favicon': None, 'picture': None, 'authors': [], 'chapter_preview_only': [], 'license': None, 'links': {'manual': { 'title': 'Manual', 'target': 'manual.html', } } } def __init__(self, user_conf=None): """Build from user_conf""" self.conf = self._get_config(user_conf) self.conf['template_path'] = Path(self.conf['theme']['dir'], 'templates') for key in 'nb', 'theme': self.conf[key]['path'] = Path(self.conf[key]['dir']) self.conf[key]['material_paths'] = [ self.conf[key]['path'] / Path(d) for d in self.conf[key]['material'] ] self.conf['pages']['path'] = Path(self.conf['pages']['dir']) self.conf['output_path'] = Path(self.conf['output_dir']) if self.conf['local_reveal']: nbcourse_libdir = Path(__file__).parents[1] self.conf['reveal_path'] = nbcourse_libdir / Path('reveal.js') self.notebooks = tuple(self.conf['nb']['path'].glob('*-*.ipynb')) if user_conf and not self.notebooks: log.error(""" No notebooks found! 1. Check 'nb:dir:' field in nbcourse.yml file. 2. Ensure that your notebooks are named using the pattern: - "01-my_first_chapter_name.ipynb", - "02-my_second_chapter_name.ipynb", - etc. """) sys.exit() if self.conf['book']['file']: self.titlepage_path = self.conf['pages']['path'] / \ self.conf['book']['titlepage'] self.book = self.conf['output_path'] / self.conf['book']['file'] else: self.titlepage_path = None self.book = None self.md_page_paths = list(self.conf['pages']['path'].glob('*.md')) self.html_pages = self._get_pages() # homepage and documentation pages self.theme_files = [file for file in self.conf['theme']['path'].glob('**/*') if file.is_file()] self.htmls = [] # html export of notebooks self.slides = [] # reveal slide export of notebooks self.material = [] # additional material (pictures, etc.) self.chapter_zips = [] self.executed_notebooks = [self._get_executed_nb_path(notebook) for notebook in self.notebooks] self.zip_file = self.conf['output_path'] / \ Path(self.conf['slug_title']).with_suffix('.zip') # Instantiate notebook exporter self.html_exporter = HTMLExporter() # Instantiate slide exporter self.slide_exporter = SlidesExporter() reveal_url = \ "https://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.7.0" if self.conf['local_reveal']: self.slide_exporter.reveal_url_prefix = 'reveal.js' else: self.slide_exporter.reveal_url_prefix = reveal_url def _get_config(self, user_conf): """Get conf dict from default and user_conf""" def sanitize(user_conf_dict: dict): """ Sanitize user_conf_dict: remove first level keys that are not present in default dict """ for k in user_conf_dict: if k not in conf.keys(): log.warning(f'Unknown configuration parameter {k}: ' 'ignored.') del(k) conf = self.default_conf.copy() if type(user_conf) is PosixPath: # Load user conf as file and update default conf config_file = user_conf # Load YAML file as dict try: with open(config_file, 'r') as f: user_conf_dict = yaml.safe_load(f) except FileNotFoundError as e: log.error(f'''"{e.filename}" file not found. Consider initializing an nbcourse project with: nbcourse --init\ ''') sys.exit() sanitize(user_conf_dict) update_dict(conf, user_conf_dict) conf['project_path'] = config_file.absolute().parent conf['config_file'] = config_file else: if type(user_conf) is dict: # Load user conf as dict and update default conf sanitize(user_conf) update_dict(conf, user_conf) else: # Only default config is loaded (used for tests only) log.debug("Default configuration is used") conf['project_path'] = Path.cwd() conf['config_file'] = None return conf def _get_pages(self): """get a list of target html pages from markdown source""" # Start html_pages list with index.html html_pages = [self.conf['output_path'] / self.conf['pages']['home']] for path in self.md_page_paths: html_pages.append(self.conf['output_path'] / Path(path.name).with_suffix('.html')) return html_pages def build_pages(self): """ Build a mini website using html templating and markdown conversion """ # List mardkown files md_page_paths = list(self.conf['pages']['path'].glob('*.md')) # Instanciate all markdown pages to be rendered md_pages = [MarkdownPage(self, md_page_path) for md_page_path in md_page_paths] def render_children(parent): """Recursive function to render children pages from parent page""" children = (md_page for md_page in md_pages if md_page.parent_name == parent.name) for child in children: # Render markdown pages child.render(parent) render_children(child) # First build homepage homepage = HomePage(self) homepage.render() # Then build children pages render_children(homepage) def build_book(self): """Build a pdf book using bookbook""" book = Book(self.conf) book.build() def task_output_dir(self): """Create empty output directory""" return { 'targets': [self.conf['output_path']], 'actions': [(create_folder, [self.conf['output_path']])], 'uptodate': [self.conf['output_path'].is_dir], 'clean': True, } def get_src_dst_paths(self, src_path: Path): """Return a tuple of file paths for source and destination""" if src_path.is_dir(): files = get_file_list(src_path) src_files = [src_path / file for file in files] dst_path = self.conf['output_path'] / Path(src_path.name) dst_files = [dst_path / file for file in files] else: # dealing with single file, preserve directory tree src_files = [src_path] rpath = src_path.relative_to(self.conf['nb']['path']) dst_path = self.conf['output_path'] / rpath dst_files = [dst_path] return src_files, dst_path, dst_files def task_copy_material(self): """Copy notebook and theme material to output directory""" for parent in 'nb', 'theme': for src_path in self.conf[parent]['material_paths']: src_files, dst_path, dst_files = self.get_src_dst_paths( src_path) self.material += dst_files yield { 'name': dst_path, 'file_dep': src_files, 'targets': dst_files, 'actions': [(update_material, (src_path, dst_path))], 'clean': [(clean_tree, (dst_path,))], } def task_copy_reveal(self): """Copy reveal.js to output directory""" if self.conf['local_reveal']: src_path = self.conf['reveal_path'] src_files, dst_path, dst_files = self.get_src_dst_paths(src_path) return { 'file_dep': src_files, 'targets': dst_files, 'actions': [(update_material, (src_path, dst_path))], 'clean': [(clean_tree, (dst_path,))], } else: return { 'uptodate': [True], 'actions': [] } def execute_notebook(self, notebook: Path, executed_notebook: Path): """Execute notebook and write result to executed_notebook file""" with open(notebook) as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(timeout=self.conf['nb']['timeout'], allow_errors=True) ep.preprocess(nb, {'metadata': {'path': self.conf['nb']['path']}}) with open(executed_notebook, 'w', encoding='utf-8') as f: nbformat.write(nb, f) def _get_executed_nb_path(self, notebook) -> Path: """Return executed notebook path""" return self.conf['output_path'] / Path(notebook.name) def task_execute_notebooks(self): """Write executed notebooks to output directory""" for notebook in self.notebooks: executed_notebook = self._get_executed_nb_path(notebook) yield { 'name': executed_notebook, 'file_dep': [notebook], 'task_dep': ['copy_material'], 'targets': [executed_notebook], 'clean': True, 'actions': [(self.execute_notebook, (notebook, executed_notebook))], } def convert_notebook(self, notebook: Path, output_file: Path, to='html'): """Convert notebook to output_file""" log.debug(f"Converting {notebook} to {output_file}") if to == 'slide': exporter = self.slide_exporter else: exporter = self.html_exporter with open(notebook) as f: nb = nbformat.read(f, as_version=4) body, resources = exporter.from_notebook_node(nb) with open(output_file, 'w', encoding='utf-8') as f: f.write(body) def task_convert_to_html(self): """Convert executed notebook to html page""" for notebook in self.notebooks: executed_notebook = self._get_executed_nb_path(notebook) html = executed_notebook.with_suffix('.html') self.htmls.append(html) yield { 'name': html, 'file_dep': [executed_notebook], 'targets': [html], 'clean': True, 'actions': [(self.convert_notebook, (executed_notebook, html))], } def task_convert_to_slides(self): """Convert executed notebook to reveal slides""" for notebook in self.notebooks: executed_notebook = self._get_executed_nb_path(notebook) slide = executed_notebook.with_suffix('.slides.html') self.slides.append(slide) yield { 'name': slide, 'file_dep': [executed_notebook], 'uptodate': [config_changed(str(self.conf['local_reveal']))], 'targets': [slide], 'clean': True, 'actions': [(self.convert_notebook, (executed_notebook, slide, 'slide'))], } def task_zip_chapters(self): """Build zip archives for single chapter downloads""" for notebook in self.notebooks: zip_file = self._get_executed_nb_path(notebook).with_suffix('.zip') paths_to_zip = [notebook] for path in self.conf['nb']['material_paths']: paths_to_zip += get_file_list(path, relative=False) self.chapter_zips.append(zip_file.name) yield { 'name': zip_file, 'file_dep': paths_to_zip, 'task_dep': ['copy_material'], 'targets': [zip_file], 'clean': True, 'actions': [(zip_files, (zip_file, paths_to_zip))], } def task_build_pages(self): """Build html pages""" deps = self.executed_notebooks + self.md_page_paths + self.theme_files if self.conf['config_file']: deps.append(self.conf['config_file']) return { 'file_dep': deps, 'task_dep': ['copy_material'], 'targets': self.html_pages, 'clean': True, 'actions': [self.build_pages], } def task_build_book(self): """Build pdf book""" if self.book: return { 'file_dep': self.executed_notebooks + [self.titlepage_path], 'targets': [self.book], 'clean': True, 'actions': [self.build_book], } else: return { 'uptodate': [True], 'actions': [] } def task_zip_archive(self): """Build a single zip archive for all material""" paths_to_zip = self.executed_notebooks + \ self.html_pages + self.htmls + self.slides + self.material if self.book: paths_to_zip.append(self.book) return { 'file_dep': paths_to_zip, 'targets': [self.zip_file], 'clean': True, 'actions': [(zip_files, (self.zip_file, paths_to_zip), {'ref_to_arc': (self.conf['output_path'], self.conf['slug_title'])})], } def build(self, args: list = None): """Build course using Doit""" if args is None: args = [] doit_config = {} # TODO: implement verbosity option return MyDoitMain(ClassTaskLoader(self), extra_config=doit_config).run(args)