#!/usr/bin/env python3 """ Define a NbCourse object containing: - class methods for building html pages and pdf book - Doit tasks defined in task_*() functions """ from pathlib import Path import yaml import sys from doit.tools import create_folder, config_changed from .utils import update_material, clean_tree, get_file_list, update_dict, \ zip_files from .pages import HomePage, MarkdownPage from .book import Book import nbformat from nbconvert.preprocessors import ExecutePreprocessor from nbconvert import HTMLExporter, SlidesExporter from .mydoit import MyDoitMain, ClassTaskLoader import logging log = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) DEFAULT_CONFIG_FILE = "nbcourse.yml" class NbCourse: default_conf = { 'theme': { 'dir': 'theme/default', 'material': ('css', 'img') }, 'nb': { 'dir': 'notebooks', 'material': ('fig', 'exos') }, 'pages': { 'dir': 'pages', 'titlepage': 'titlepage.tex', 'home': 'index.html' }, 'bookbook_filename': 'bookbook.tex', 'local_reveal': False, 'slug_title': 'course', 'output_dir': 'build', 'title': 'A course', } def __init__(self, config_file: Path = None): self.conf = self.default_conf.copy() if config_file: update_dict(self.conf, self._get_user_config(config_file)) self.conf['project_path'] = Path(config_file).absolute().parent else: # Only default config is loaded (only useful for tests) self.conf['project_path'] = Path.cwd() self.conf['template_path'] = Path(self.conf['theme']['dir'], 'templates') for key in 'nb', 'theme': self.conf[key]['path'] = Path(self.conf[key]['dir']) self.conf[key]['material_paths'] = [ self.conf[key]['path'] / Path(d) for d in self.conf[key]['material'] ] self.conf['pages']['path'] = Path(self.conf['pages']['dir']) self.conf['output_path'] = Path(self.conf['output_dir']) if self.conf['local_reveal']: libdir = Path(sys.argv[0]).parent self.conf['reveal_path'] = libdir / Path('reveal.js') self.conf['book_title'] = Path(self.conf['slug_title']).with_suffix( '.pdf') self.notebooks = tuple(self.conf['nb']['path'].glob('*-*.ipynb')) self.book = self.conf['output_path'] / self.conf['book_title'] self.md_page_paths = list(self.conf['pages']['path'].glob('*.md')) self.html_pages = self._get_pages() # homepage and documentation pages self.theme_files = [file for file in self.conf['theme']['path'].glob('**/*') if file.is_file()] self.htmls = [] # html export of notebooks self.slides = [] # reveal slide export of notebooks self.material = [] # additional material (pictures, etc.) self.chapter_zips = [] self.executed_notebooks = [self.get_executed_notebook(notebook) for notebook in self.notebooks] self.zip_file = self.conf['output_path'] / \ Path(self.conf['slug_title']).with_suffix('.zip') # Instantiate notebook exporter self.html_exporter = HTMLExporter() # Instantiate slide exporter self.slide_exporter = SlidesExporter() reveal_url = \ "https://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.7.0" if self.conf['local_reveal']: self.slide_exporter.reveal_url_prefix = 'reveal.js' else: self.slide_exporter.reveal_url_prefix = reveal_url def _get_pages(self): """get a list of target html pages from markdown source""" # Start html_pages list with index.html html_pages = [self.conf['output_path'] / self.conf['pages']['home']] for path in self.md_page_paths: html_pages.append(self.conf['output_path'] / Path(path.name).with_suffix('.html')) return html_pages @staticmethod def _get_user_config(config_file: Path) -> dict: """ Parse yaml file to build the corresponding configuration dictionary """ with open(config_file, 'r') as f: return yaml.safe_load(f) def build_pages(self): """ Build a mini website using html templating and markdown conversion """ # List mardkown files md_page_paths = list(self.conf['pages']['path'].glob('*.md')) # Instanciate all markdown pages to be rendered md_pages = [MarkdownPage(self, md_page_path) for md_page_path in md_page_paths] def render_children(parent): """Recursive function to render children pages from parent page""" children = (md_page for md_page in md_pages if md_page.parent_name == parent.name) for child in children: # Render markdown pages child.render(parent) render_children(child) # First build homepage homepage = HomePage(self) homepage.render() # Then build children pages render_children(homepage) def build_book(self): """Build a pdf book using bookbook""" book = Book(self.conf) book.build() def task_output_dir(self): """Create empty output directory""" return { 'targets': [self.conf['output_path']], 'actions': [(create_folder, [self.conf['output_path']])], 'uptodate': [self.conf['output_path'].is_dir], 'clean': True, } def get_src_dst_paths(self, src_path: Path): """Return a tuple of file paths for source and destination""" files = get_file_list(src_path) src_files = [src_path / file for file in files] dst_path = self.conf['output_path'] / Path(src_path.name) dst_files = [dst_path / file for file in files] return src_files, dst_path, dst_files def task_copy_material(self): """Copy notebook and theme material to output directory""" for parent in 'nb', 'theme': for src_path in self.conf[parent]['material_paths']: src_files, dst_path, dst_files = self.get_src_dst_paths( src_path) self.material += dst_files yield { 'name': dst_path, 'file_dep': src_files, 'targets': dst_files, 'actions': [(update_material, (src_path, dst_path))], 'clean': [(clean_tree, (dst_path,))], } def task_copy_reveal(self): """Copy reveal.js to output directory""" if self.conf['local_reveal']: src_path = self.conf['reveal_path'] src_files, dst_path, dst_files = self.get_src_dst_paths(src_path) return { 'file_dep': src_files, 'targets': dst_files, 'actions': [(update_material, (src_path, dst_path))], 'clean': [(clean_tree, (dst_path,))], } else: return { 'uptodate': [True], 'actions': [] } def execute_notebook(self, notebook: Path, executed_notebook: Path): """Execute notebook and write result to executed_notebook file""" with open(notebook) as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(timeout=60, kernel_name='python3', allow_errors=True) ep.preprocess(nb, {'metadata': {'path': self.conf['nb']['path']}}) with open(executed_notebook, 'w', encoding='utf-8') as f: nbformat.write(nb, f) def get_executed_notebook(self, notebook) -> Path: """Return executed notebook path""" return self.conf['output_path'] / Path(notebook.name) def task_execute_notebooks(self): """Write executed notebooks to output directory""" for notebook in self.notebooks: executed_notebook = self.get_executed_notebook(notebook) yield { 'name': executed_notebook, 'file_dep': [notebook], 'task_dep': ['copy_material'], 'targets': [executed_notebook], 'clean': True, 'actions': [(self.execute_notebook, (notebook, executed_notebook))], } def convert_notebook(self, notebook: Path, output_file: Path, to='html'): """Convert notebook to output_file""" if to == 'slide': exporter = self.slide_exporter else: exporter = self.html_exporter with open(notebook) as f: nb = nbformat.read(f, as_version=4) body, resources = exporter.from_notebook_node(nb) with open(output_file, 'w', encoding='utf-8') as f: f.write(body) def task_convert_to_html(self): """Convert executed notebook to html page""" for notebook in self.notebooks: executed_notebook = self.get_executed_notebook(notebook) html = executed_notebook.with_suffix('.html') self.htmls.append(html) yield { 'name': html, 'file_dep': [executed_notebook], 'targets': [html], 'clean': True, 'actions': [(self.convert_notebook, (notebook, html))], } def task_convert_to_slides(self): """Convert executed notebook to reveal slides""" for notebook in self.notebooks: executed_notebook = self.get_executed_notebook(notebook) slide = executed_notebook.with_suffix('.slides.html') self.slides.append(slide) yield { 'name': slide, 'file_dep': [executed_notebook], 'uptodate': [config_changed(str(self.conf['local_reveal']))], 'targets': [slide], 'clean': True, 'actions': [(self.convert_notebook, (notebook, slide, 'slide'))], } def task_zip_chapters(self): """Build zip archives for single chapter downloads""" for notebook in self.notebooks: zip_file = self.get_executed_notebook(notebook).with_suffix('.zip') paths_to_zip = [notebook] for path in self.conf['nb']['material_paths']: paths_to_zip += get_file_list(path, relative=False) self.chapter_zips.append(zip_file.name) yield { 'name': zip_file, 'file_dep': paths_to_zip, 'task_dep': ['copy_material'], 'targets': [zip_file], 'clean': True, 'actions': [(zip_files, (zip_file, paths_to_zip))], } def task_build_pages(self): """Build html pages""" return { 'file_dep': self.md_page_paths + self.theme_files, 'task_dep': ['copy_material'], 'targets': self.html_pages, 'clean': True, 'actions': [self.build_pages], } def task_build_book(self): """Build pdf book""" return { 'file_dep': self.executed_notebooks, 'targets': [self.book], 'clean': True, 'actions': [self.build_book], } def task_zip_archive(self): """Build a single zip archive for all material""" paths_to_zip = [self.book] + self.executed_notebooks + \ self.html_pages + self.htmls + self.slides + self.material return { 'file_dep': paths_to_zip, 'targets': [self.zip_file], 'clean': True, 'actions': [(zip_files, (self.zip_file, paths_to_zip), {'ref_to_arc': (self.conf['output_path'], self.conf['slug_title'])})], } def build(self, args: list = None): """Build course using Doit""" if args is None: args = [] doit_config = {} # TODO: implement verbosity option sys.exit(MyDoitMain(ClassTaskLoader(self), extra_config=doit_config).run(args))