nbcourse.py 15.5 KB
Newer Older
Matthieu Boileau's avatar
Matthieu Boileau committed
1 2
#!/usr/bin/env python3
"""
Matthieu Boileau's avatar
Matthieu Boileau committed
3 4 5
Define a NbCourse object containing:
    - class methods for building html pages and pdf book
    - Doit tasks defined in task_*() functions
Matthieu Boileau's avatar
Matthieu Boileau committed
6
"""
Matthieu Boileau's avatar
Matthieu Boileau committed
7

8
from pathlib import Path, PosixPath
Matthieu Boileau's avatar
Matthieu Boileau committed
9
import yaml
Matthieu Boileau's avatar
Matthieu Boileau committed
10
from doit.tools import create_folder, config_changed
Matthieu Boileau's avatar
Matthieu Boileau committed
11
from .utils import update_material, clean_tree, get_file_list, update_dict, \
Matthieu Boileau's avatar
Matthieu Boileau committed
12
    zip_files
Matthieu Boileau's avatar
Matthieu Boileau committed
13 14
from .pages import HomePage, MarkdownPage
from .book import Book
15
import nbformat
Matthieu Boileau's avatar
Matthieu Boileau committed
16 17 18
from nbconvert.preprocessors import ExecutePreprocessor
from nbconvert import HTMLExporter, SlidesExporter
from .mydoit import MyDoitMain, ClassTaskLoader
Matthieu Boileau's avatar
Matthieu Boileau committed
19
import logging
20
import sys
21

Matthieu Boileau's avatar
Matthieu Boileau committed
22 23
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
24

25
DEFAULT_CONFIG_FILE = "nbcourse.yml"
Matthieu Boileau's avatar
Matthieu Boileau committed
26 27


28 29
class NbCourse:

Matthieu Boileau's avatar
Matthieu Boileau committed
30 31 32 33 34 35
    default_conf = {
                    'theme': {
                        'dir': 'theme/default',
                        'material': ('css', 'img')
                        },
                    'nb': {
Matthieu Boileau's avatar
Matthieu Boileau committed
36
                        'dir': '.',
37
                        'timeout': 60,
Matthieu Boileau's avatar
Fix #15  
Matthieu Boileau committed
38
                        'material': ()
Matthieu Boileau's avatar
Matthieu Boileau committed
39 40 41
                        },
                    'pages': {
                        'dir': 'pages',
Matthieu Boileau's avatar
Matthieu Boileau committed
42
                        'home': 'index.html'
Matthieu Boileau's avatar
Matthieu Boileau committed
43
                        },
Matthieu Boileau's avatar
Matthieu Boileau committed
44 45
                    'book': {
                        'titlepage': 'titlepage.tex',
Matthieu Boileau's avatar
Matthieu Boileau committed
46
                        'file': None,
Matthieu Boileau's avatar
Matthieu Boileau committed
47
                        },
Matthieu Boileau's avatar
Matthieu Boileau committed
48 49 50
                    'local_reveal': False,
                    'slug_title': 'course',
                    'output_dir': 'build',
Matthieu Boileau's avatar
Matthieu Boileau committed
51
                    'title': 'A course',
Matthieu Boileau's avatar
Matthieu Boileau committed
52 53 54 55 56 57
                    'subtitle': None,
                    'favicon': None,
                    'picture': None,
                    'authors': [],
                    'chapter_preview_only': [],
                    'license': None,
Matthieu Boileau's avatar
V2.0  
Matthieu Boileau committed
58 59 60 61 62
                    'links': {'manual': {
                                    'title': 'Manual',
                                    'target': 'manual.html',
                                    }
                              }
Matthieu Boileau's avatar
Matthieu Boileau committed
63 64
                    }

Matthieu Boileau's avatar
Matthieu Boileau committed
65 66
    def __init__(self, user_conf=None):
        """Build from user_conf"""
Matthieu Boileau's avatar
Fix #16  
Matthieu Boileau committed
67
        self.conf = self._get_config(user_conf)
Matthieu Boileau's avatar
Matthieu Boileau committed
68 69 70 71 72
        self.conf['template_path'] = Path(self.conf['theme']['dir'],
                                          'templates')
        for key in 'nb', 'theme':
            self.conf[key]['path'] = Path(self.conf[key]['dir'])
            self.conf[key]['material_paths'] = [
Matthieu Boileau's avatar
Matthieu Boileau committed
73 74
                self.conf[key]['path'] / Path(d)
                for d in self.conf[key]['material']
Matthieu Boileau's avatar
Matthieu Boileau committed
75 76 77 78
                ]
        self.conf['pages']['path'] = Path(self.conf['pages']['dir'])
        self.conf['output_path'] = Path(self.conf['output_dir'])
        if self.conf['local_reveal']:
Matthieu Boileau's avatar
Matthieu Boileau committed
79 80
            nbcourse_libdir = Path(__file__).parents[1]
            self.conf['reveal_path'] = nbcourse_libdir / Path('reveal.js')
Matthieu Boileau's avatar
Matthieu Boileau committed
81
        self.notebooks = tuple(self.conf['nb']['path'].glob('*-*.ipynb'))
82
        if user_conf and not self.notebooks:
Matthieu Boileau's avatar
V2.0  
Matthieu Boileau committed
83 84 85 86 87 88 89 90
            log.error("""
No notebooks found!
1. Check 'nb:dir:' field in nbcourse.yml file.
2. Ensure that your notebooks are named using the pattern:
   - "01-my_first_chapter_name.ipynb",
   - "02-my_second_chapter_name.ipynb",
   - etc.
""")
91
            sys.exit()
Matthieu Boileau's avatar
Matthieu Boileau committed
92
        if self.conf['book']['file']:
Matthieu Boileau's avatar
Matthieu Boileau committed
93 94
            self.titlepage_path = self.conf['pages']['path'] / \
                self.conf['book']['titlepage']
Matthieu Boileau's avatar
Matthieu Boileau committed
95
            self.book = self.conf['output_path'] / self.conf['book']['file']
Matthieu Boileau's avatar
Matthieu Boileau committed
96 97 98
        else:
            self.titlepage_path = None
            self.book = None
Matthieu Boileau's avatar
Matthieu Boileau committed
99
        self.md_page_paths = list(self.conf['pages']['path'].glob('*.md'))
Matthieu Boileau's avatar
Matthieu Boileau committed
100 101 102 103 104 105 106 107 108
        self.html_pages = self._get_pages()  # homepage and documentation pages
        self.theme_files = [file for file in
                            self.conf['theme']['path'].glob('**/*')
                            if file.is_file()]
        self.htmls = []  # html export of notebooks
        self.slides = []  # reveal slide export of notebooks
        self.material = []  # additional material (pictures, etc.)
        self.chapter_zips = []

Matthieu Boileau's avatar
Fix #15  
Matthieu Boileau committed
109
        self.executed_notebooks = [self._get_executed_nb_path(notebook)
Matthieu Boileau's avatar
Matthieu Boileau committed
110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125
                                   for notebook in self.notebooks]
        self.zip_file = self.conf['output_path'] / \
            Path(self.conf['slug_title']).with_suffix('.zip')

        # Instantiate notebook exporter
        self.html_exporter = HTMLExporter()

        # Instantiate slide exporter
        self.slide_exporter = SlidesExporter()
        reveal_url = \
            "https://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.7.0"
        if self.conf['local_reveal']:
            self.slide_exporter.reveal_url_prefix = 'reveal.js'
        else:
            self.slide_exporter.reveal_url_prefix = reveal_url

Matthieu Boileau's avatar
Fix #16  
Matthieu Boileau committed
126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
    def _get_config(self, user_conf):
        """Get conf dict from default and user_conf"""

        def sanitize(user_conf_dict: dict):
            """
            Sanitize user_conf_dict: remove first level keys that are
            not present in default dict
            """
            for k in user_conf_dict:
                if k not in conf.keys():
                    log.warning(f'Unknown configuration parameter {k}: '
                                'ignored.')
                    del(k)

        conf = self.default_conf.copy()
141
        if type(user_conf) is PosixPath:
Matthieu Boileau's avatar
Fix #16  
Matthieu Boileau committed
142 143 144
            # Load user conf as file and update default conf
            config_file = user_conf
            # Load YAML file as dict
145 146 147 148
            try:
                with open(config_file, 'r') as f:
                    user_conf_dict = yaml.safe_load(f)
            except FileNotFoundError as e:
Matthieu Boileau's avatar
V2.0  
Matthieu Boileau committed
149 150 151 152
                log.error(f'''"{e.filename}" file not found.
Consider initializing an nbcourse project with:
nbcourse --init\
''')
153
                sys.exit()
Matthieu Boileau's avatar
Fix #16  
Matthieu Boileau committed
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169
            sanitize(user_conf_dict)
            update_dict(conf, user_conf_dict)
            conf['project_path'] = config_file.absolute().parent
            conf['config_file'] = config_file
        else:
            if type(user_conf) is dict:
                # Load user conf as dict and update default conf
                sanitize(user_conf)
                update_dict(conf, user_conf)
            else:
                # Only default config is loaded (used for tests only)
                log.debug("Default configuration is used")
            conf['project_path'] = Path.cwd()
            conf['config_file'] = None
        return conf

Matthieu Boileau's avatar
Matthieu Boileau committed
170 171 172
    def _get_pages(self):
        """get a list of target html pages from markdown source"""
        # Start html_pages list with index.html
Matthieu Boileau's avatar
Matthieu Boileau committed
173 174
        html_pages = [self.conf['output_path'] / self.conf['pages']['home']]
        for path in self.md_page_paths:
Matthieu Boileau's avatar
Matthieu Boileau committed
175
            html_pages.append(self.conf['output_path'] /
Matthieu Boileau's avatar
Matthieu Boileau committed
176
                              Path(path.name).with_suffix('.html'))
Matthieu Boileau's avatar
Matthieu Boileau committed
177
        return html_pages
178 179

    def build_pages(self):
Matthieu Boileau's avatar
Matthieu Boileau committed
180 181 182
        """
        Build a mini website using html templating and markdown conversion
        """
183

Matthieu Boileau's avatar
Matthieu Boileau committed
184 185 186 187 188 189 190
        # List mardkown files
        md_page_paths = list(self.conf['pages']['path'].glob('*.md'))
        # Instanciate all markdown pages to be rendered
        md_pages = [MarkdownPage(self, md_page_path)
                    for md_page_path in md_page_paths]

        def render_children(parent):
191
            """Recursive function to render children pages from parent page"""
Matthieu Boileau's avatar
Matthieu Boileau committed
192 193 194 195 196 197
            children = (md_page for md_page in md_pages
                        if md_page.parent_name == parent.name)
            for child in children:
                # Render markdown pages
                child.render(parent)
                render_children(child)
198 199

        # First build homepage
Matthieu Boileau's avatar
Matthieu Boileau committed
200 201
        homepage = HomePage(self)
        homepage.render()
202

203
        # Then build children pages
Matthieu Boileau's avatar
Matthieu Boileau committed
204
        render_children(homepage)
205

Matthieu Boileau's avatar
Matthieu Boileau committed
206
    def build_book(self):
207
        """Build a pdf book using bookbook"""
Matthieu Boileau's avatar
Matthieu Boileau committed
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222

        book = Book(self.conf)
        book.build()

    def task_output_dir(self):
        """Create empty output directory"""
        return {
            'targets': [self.conf['output_path']],
            'actions': [(create_folder, [self.conf['output_path']])],
            'uptodate': [self.conf['output_path'].is_dir],
            'clean': True,
        }

    def get_src_dst_paths(self, src_path: Path):
        """Return a tuple of file paths for source and destination"""
223 224 225 226 227
        if src_path.is_dir():
            files = get_file_list(src_path)
            src_files = [src_path / file for file in files]
            dst_path = self.conf['output_path'] / Path(src_path.name)
            dst_files = [dst_path / file for file in files]
Matthieu Boileau's avatar
Matthieu Boileau committed
228
        else:  # dealing with single file, preserve directory tree
229
            src_files = [src_path]
Matthieu Boileau's avatar
V2.0  
Matthieu Boileau committed
230
            rpath = src_path.relative_to(self.conf['nb']['path'])
231
            dst_path = self.conf['output_path'] / rpath
232
            dst_files = [dst_path]
Matthieu Boileau's avatar
Matthieu Boileau committed
233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260
        return src_files, dst_path, dst_files

    def task_copy_material(self):
        """Copy notebook and theme material to output directory"""
        for parent in 'nb', 'theme':
            for src_path in self.conf[parent]['material_paths']:
                src_files, dst_path, dst_files = self.get_src_dst_paths(
                    src_path)
                self.material += dst_files
                yield {
                    'name': dst_path,
                    'file_dep': src_files,
                    'targets': dst_files,
                    'actions': [(update_material, (src_path, dst_path))],
                    'clean': [(clean_tree, (dst_path,))],
                }

    def task_copy_reveal(self):
        """Copy reveal.js to output directory"""
        if self.conf['local_reveal']:
            src_path = self.conf['reveal_path']
            src_files, dst_path, dst_files = self.get_src_dst_paths(src_path)
            return {
                'file_dep': src_files,
                'targets': dst_files,
                'actions': [(update_material, (src_path, dst_path))],
                'clean': [(clean_tree, (dst_path,))],
            }
Matthieu Boileau's avatar
Matthieu Boileau committed
261 262 263 264 265
        else:
            return {
                'uptodate': [True],
                'actions': []
            }
Matthieu Boileau's avatar
Matthieu Boileau committed
266 267 268 269 270 271

    def execute_notebook(self, notebook: Path, executed_notebook: Path):
        """Execute notebook and write result to executed_notebook file"""
        with open(notebook) as f:
            nb = nbformat.read(f, as_version=4)

272
        ep = ExecutePreprocessor(timeout=self.conf['nb']['timeout'],
Matthieu Boileau's avatar
Matthieu Boileau committed
273 274 275 276 277 278
                                 allow_errors=True)
        ep.preprocess(nb, {'metadata': {'path': self.conf['nb']['path']}})

        with open(executed_notebook, 'w', encoding='utf-8') as f:
            nbformat.write(nb, f)

Matthieu Boileau's avatar
Fix #15  
Matthieu Boileau committed
279
    def _get_executed_nb_path(self, notebook) -> Path:
Matthieu Boileau's avatar
Matthieu Boileau committed
280 281 282 283 284 285
        """Return executed notebook path"""
        return self.conf['output_path'] / Path(notebook.name)

    def task_execute_notebooks(self):
        """Write executed notebooks to output directory"""
        for notebook in self.notebooks:
Matthieu Boileau's avatar
Fix #15  
Matthieu Boileau committed
286
            executed_notebook = self._get_executed_nb_path(notebook)
Matthieu Boileau's avatar
Matthieu Boileau committed
287 288 289
            yield {
                'name': executed_notebook,
                'file_dep': [notebook],
Matthieu Boileau's avatar
Matthieu Boileau committed
290
                'task_dep': ['copy_material'],
Matthieu Boileau's avatar
Matthieu Boileau committed
291 292 293 294 295 296 297 298
                'targets': [executed_notebook],
                'clean': True,
                'actions': [(self.execute_notebook, (notebook,
                                                     executed_notebook))],
            }

    def convert_notebook(self, notebook: Path,
                         output_file: Path,
Matthieu Boileau's avatar
Matthieu Boileau committed
299
                         to='html'):
Matthieu Boileau's avatar
Matthieu Boileau committed
300
        """Convert notebook to output_file"""
Matthieu Boileau's avatar
Fix #15  
Matthieu Boileau committed
301
        log.debug(f"Converting {notebook} to {output_file}")
Matthieu Boileau's avatar
Matthieu Boileau committed
302
        if to == 'slide':
Matthieu Boileau's avatar
Matthieu Boileau committed
303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
            exporter = self.slide_exporter
        else:
            exporter = self.html_exporter

        with open(notebook) as f:
            nb = nbformat.read(f, as_version=4)

        body, resources = exporter.from_notebook_node(nb)

        with open(output_file, 'w', encoding='utf-8') as f:
            f.write(body)

    def task_convert_to_html(self):
        """Convert executed notebook to html page"""
        for notebook in self.notebooks:
Matthieu Boileau's avatar
Fix #15  
Matthieu Boileau committed
318
            executed_notebook = self._get_executed_nb_path(notebook)
Matthieu Boileau's avatar
Matthieu Boileau committed
319 320 321 322 323 324 325
            html = executed_notebook.with_suffix('.html')
            self.htmls.append(html)
            yield {
                'name': html,
                'file_dep': [executed_notebook],
                'targets': [html],
                'clean': True,
Matthieu Boileau's avatar
Fix #15  
Matthieu Boileau committed
326 327
                'actions': [(self.convert_notebook,
                             (executed_notebook, html))],
Matthieu Boileau's avatar
Matthieu Boileau committed
328 329 330 331 332
            }

    def task_convert_to_slides(self):
        """Convert executed notebook to reveal slides"""
        for notebook in self.notebooks:
Matthieu Boileau's avatar
Fix #15  
Matthieu Boileau committed
333
            executed_notebook = self._get_executed_nb_path(notebook)
Matthieu Boileau's avatar
Matthieu Boileau committed
334 335 336 337 338 339 340 341 342
            slide = executed_notebook.with_suffix('.slides.html')
            self.slides.append(slide)
            yield {
                'name': slide,
                'file_dep': [executed_notebook],
                'uptodate': [config_changed(str(self.conf['local_reveal']))],
                'targets': [slide],
                'clean': True,
                'actions': [(self.convert_notebook,
Matthieu Boileau's avatar
Fix #15  
Matthieu Boileau committed
343
                             (executed_notebook, slide, 'slide'))],
Matthieu Boileau's avatar
Matthieu Boileau committed
344 345 346
            }

    def task_zip_chapters(self):
Matthieu Boileau's avatar
Matthieu Boileau committed
347
        """Build zip archives for single chapter downloads"""
Matthieu Boileau's avatar
Matthieu Boileau committed
348 349

        for notebook in self.notebooks:
Matthieu Boileau's avatar
Fix #15  
Matthieu Boileau committed
350
            zip_file = self._get_executed_nb_path(notebook).with_suffix('.zip')
Matthieu Boileau's avatar
Matthieu Boileau committed
351 352 353 354 355 356 357
            paths_to_zip = [notebook]
            for path in self.conf['nb']['material_paths']:
                paths_to_zip += get_file_list(path, relative=False)
            self.chapter_zips.append(zip_file.name)
            yield {
                'name': zip_file,
                'file_dep': paths_to_zip,
Matthieu Boileau's avatar
Matthieu Boileau committed
358
                'task_dep': ['copy_material'],
Matthieu Boileau's avatar
Matthieu Boileau committed
359 360 361 362 363 364 365
                'targets': [zip_file],
                'clean': True,
                'actions': [(zip_files, (zip_file, paths_to_zip))],
            }

    def task_build_pages(self):
        """Build html pages"""
366
        deps = self.executed_notebooks + self.md_page_paths + self.theme_files
Matthieu Boileau's avatar
Fix #16  
Matthieu Boileau committed
367 368
        if self.conf['config_file']:
            deps.append(self.conf['config_file'])
Matthieu Boileau's avatar
Matthieu Boileau committed
369
        return {
Matthieu Boileau's avatar
Matthieu Boileau committed
370
            'file_dep': deps,
Matthieu Boileau's avatar
Matthieu Boileau committed
371
            'task_dep': ['copy_material'],
Matthieu Boileau's avatar
Matthieu Boileau committed
372 373 374 375 376 377 378
            'targets': self.html_pages,
            'clean': True,
            'actions': [self.build_pages],
        }

    def task_build_book(self):
        """Build pdf book"""
Matthieu Boileau's avatar
Matthieu Boileau committed
379 380 381 382 383 384 385 386 387 388 389 390
        if self.book:
            return {
                'file_dep': self.executed_notebooks + [self.titlepage_path],
                'targets': [self.book],
                'clean': True,
                'actions': [self.build_book],
            }
        else:
            return {
                'uptodate': [True],
                'actions': []
            }
Matthieu Boileau's avatar
Matthieu Boileau committed
391 392 393

    def task_zip_archive(self):
        """Build a single zip archive for all material"""
Matthieu Boileau's avatar
Matthieu Boileau committed
394
        paths_to_zip = self.executed_notebooks + \
Matthieu Boileau's avatar
Matthieu Boileau committed
395
            self.html_pages + self.htmls + self.slides + self.material
Matthieu Boileau's avatar
Matthieu Boileau committed
396 397
        if self.book:
            paths_to_zip.append(self.book)
Matthieu Boileau's avatar
Matthieu Boileau committed
398 399 400 401 402
        return {
            'file_dep': paths_to_zip,
            'targets': [self.zip_file],
            'clean': True,
            'actions': [(zip_files, (self.zip_file, paths_to_zip),
Matthieu Boileau's avatar
Matthieu Boileau committed
403 404
                         {'ref_to_arc': (self.conf['output_path'],
                                         self.conf['slug_title'])})],
Matthieu Boileau's avatar
Matthieu Boileau committed
405 406
        }

Matthieu Boileau's avatar
Matthieu Boileau committed
407 408 409 410
    def build(self, args: list = None):
        """Build course using Doit"""
        if args is None:
            args = []
Matthieu Boileau's avatar
Matthieu Boileau committed
411
        doit_config = {}  # TODO: implement verbosity option
Matthieu Boileau's avatar
Matthieu Boileau committed
412 413
        return MyDoitMain(ClassTaskLoader(self),
                          extra_config=doit_config).run(args)