nbcourse.py 14.7 KB
Newer Older
Matthieu Boileau's avatar
Matthieu Boileau committed
1
2
#!/usr/bin/env python3
"""
Matthieu Boileau's avatar
Matthieu Boileau committed
3
4
5
Define a NbCourse object containing:
    - class methods for building html pages and pdf book
    - Doit tasks defined in task_*() functions
Matthieu Boileau's avatar
Matthieu Boileau committed
6
"""
Matthieu Boileau's avatar
Matthieu Boileau committed
7

8
from pathlib import Path, PosixPath
Matthieu Boileau's avatar
Matthieu Boileau committed
9
import yaml
Matthieu Boileau's avatar
Matthieu Boileau committed
10
import sys
Matthieu Boileau's avatar
Matthieu Boileau committed
11
from doit.tools import create_folder, config_changed
Matthieu Boileau's avatar
Matthieu Boileau committed
12
from .utils import update_material, clean_tree, get_file_list, update_dict, \
Matthieu Boileau's avatar
Matthieu Boileau committed
13
    zip_files
Matthieu Boileau's avatar
Matthieu Boileau committed
14
15
from .pages import HomePage, MarkdownPage
from .book import Book
16
import nbformat
Matthieu Boileau's avatar
Matthieu Boileau committed
17
18
19
from nbconvert.preprocessors import ExecutePreprocessor
from nbconvert import HTMLExporter, SlidesExporter
from .mydoit import MyDoitMain, ClassTaskLoader
Matthieu Boileau's avatar
Matthieu Boileau committed
20
import logging
21

Matthieu Boileau's avatar
Matthieu Boileau committed
22
23
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
24

25
DEFAULT_CONFIG_FILE = "nbcourse.yml"
Matthieu Boileau's avatar
Matthieu Boileau committed
26
27


28
29
class NbCourse:

Matthieu Boileau's avatar
Matthieu Boileau committed
30
31
32
33
34
35
36
    default_conf = {
                    'theme': {
                        'dir': 'theme/default',
                        'material': ('css', 'img')
                        },
                    'nb': {
                        'dir': 'notebooks',
Matthieu Boileau's avatar
Fix #15    
Matthieu Boileau committed
37
                        'material': ()
Matthieu Boileau's avatar
Matthieu Boileau committed
38
39
40
                        },
                    'pages': {
                        'dir': 'pages',
Matthieu Boileau's avatar
Matthieu Boileau committed
41
                        'home': 'index.html'
Matthieu Boileau's avatar
Matthieu Boileau committed
42
                        },
Matthieu Boileau's avatar
Matthieu Boileau committed
43
44
                    'book': {
                        'titlepage': 'titlepage.tex',
Matthieu Boileau's avatar
Matthieu Boileau committed
45
                        'file': None,
Matthieu Boileau's avatar
Matthieu Boileau committed
46
                        },
Matthieu Boileau's avatar
Matthieu Boileau committed
47
48
49
                    'local_reveal': False,
                    'slug_title': 'course',
                    'output_dir': 'build',
Matthieu Boileau's avatar
Matthieu Boileau committed
50
                    'title': 'A course',
Matthieu Boileau's avatar
Matthieu Boileau committed
51
52
53
54
55
56
                    'subtitle': None,
                    'favicon': None,
                    'picture': None,
                    'authors': [],
                    'chapter_preview_only': [],
                    'license': None,
Matthieu Boileau's avatar
Matthieu Boileau committed
57
58
59
60
61
62
63
64
                    'links': [{
                        'title': 'Manual',
                        'target': 'manual.html',
                        'icon': {
                            'path': 'img/Infobox_info_icon.svg',
                            'width': '35px',
                            }
                        }]
Matthieu Boileau's avatar
Matthieu Boileau committed
65
66
                    }

Matthieu Boileau's avatar
Matthieu Boileau committed
67
68
    def __init__(self, user_conf=None):
        """Build from user_conf"""
Matthieu Boileau's avatar
Fix #16    
Matthieu Boileau committed
69
        self.conf = self._get_config(user_conf)
Matthieu Boileau's avatar
Matthieu Boileau committed
70
71
72
73
74
        self.conf['template_path'] = Path(self.conf['theme']['dir'],
                                          'templates')
        for key in 'nb', 'theme':
            self.conf[key]['path'] = Path(self.conf[key]['dir'])
            self.conf[key]['material_paths'] = [
Matthieu Boileau's avatar
Matthieu Boileau committed
75
76
                self.conf[key]['path'] / Path(d)
                for d in self.conf[key]['material']
Matthieu Boileau's avatar
Matthieu Boileau committed
77
78
79
80
81
82
83
                ]
        self.conf['pages']['path'] = Path(self.conf['pages']['dir'])
        self.conf['output_path'] = Path(self.conf['output_dir'])
        if self.conf['local_reveal']:
            libdir = Path(sys.argv[0]).parent
            self.conf['reveal_path'] = libdir / Path('reveal.js')
        self.notebooks = tuple(self.conf['nb']['path'].glob('*-*.ipynb'))
Matthieu Boileau's avatar
Matthieu Boileau committed
84

Matthieu Boileau's avatar
Matthieu Boileau committed
85
        if self.conf['book']['file']:
Matthieu Boileau's avatar
Matthieu Boileau committed
86
87
            self.titlepage_path = self.conf['pages']['path'] / \
                self.conf['book']['titlepage']
Matthieu Boileau's avatar
Matthieu Boileau committed
88
            self.book = self.conf['output_path'] / self.conf['book']['file']
Matthieu Boileau's avatar
Matthieu Boileau committed
89
90
91
        else:
            self.titlepage_path = None
            self.book = None
Matthieu Boileau's avatar
Matthieu Boileau committed
92
        self.md_page_paths = list(self.conf['pages']['path'].glob('*.md'))
Matthieu Boileau's avatar
Matthieu Boileau committed
93
94
95
96
97
98
99
100
101
        self.html_pages = self._get_pages()  # homepage and documentation pages
        self.theme_files = [file for file in
                            self.conf['theme']['path'].glob('**/*')
                            if file.is_file()]
        self.htmls = []  # html export of notebooks
        self.slides = []  # reveal slide export of notebooks
        self.material = []  # additional material (pictures, etc.)
        self.chapter_zips = []

Matthieu Boileau's avatar
Fix #15    
Matthieu Boileau committed
102
        self.executed_notebooks = [self._get_executed_nb_path(notebook)
Matthieu Boileau's avatar
Matthieu Boileau committed
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
                                   for notebook in self.notebooks]
        self.zip_file = self.conf['output_path'] / \
            Path(self.conf['slug_title']).with_suffix('.zip')

        # Instantiate notebook exporter
        self.html_exporter = HTMLExporter()

        # Instantiate slide exporter
        self.slide_exporter = SlidesExporter()
        reveal_url = \
            "https://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.7.0"
        if self.conf['local_reveal']:
            self.slide_exporter.reveal_url_prefix = 'reveal.js'
        else:
            self.slide_exporter.reveal_url_prefix = reveal_url

Matthieu Boileau's avatar
Fix #16    
Matthieu Boileau committed
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
    def _get_config(self, user_conf):
        """Get conf dict from default and user_conf"""

        def sanitize(user_conf_dict: dict):
            """
            Sanitize user_conf_dict: remove first level keys that are
            not present in default dict
            """
            for k in user_conf_dict:
                if k not in conf.keys():
                    log.warning(f'Unknown configuration parameter {k}: '
                                'ignored.')
                    del(k)

        conf = self.default_conf.copy()
134
        if type(user_conf) is PosixPath:
Matthieu Boileau's avatar
Fix #16    
Matthieu Boileau committed
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
            # Load user conf as file and update default conf
            config_file = user_conf
            # Load YAML file as dict
            with open(config_file, 'r') as f:
                user_conf_dict = yaml.safe_load(f)
            sanitize(user_conf_dict)
            update_dict(conf, user_conf_dict)
            conf['project_path'] = config_file.absolute().parent
            conf['config_file'] = config_file
        else:
            if type(user_conf) is dict:
                # Load user conf as dict and update default conf
                sanitize(user_conf)
                update_dict(conf, user_conf)
            else:
                # Only default config is loaded (used for tests only)
                log.debug("Default configuration is used")
            conf['project_path'] = Path.cwd()
            conf['config_file'] = None
        return conf

Matthieu Boileau's avatar
Matthieu Boileau committed
156
157
158
    def _get_pages(self):
        """get a list of target html pages from markdown source"""
        # Start html_pages list with index.html
Matthieu Boileau's avatar
Matthieu Boileau committed
159
160
        html_pages = [self.conf['output_path'] / self.conf['pages']['home']]
        for path in self.md_page_paths:
Matthieu Boileau's avatar
Matthieu Boileau committed
161
            html_pages.append(self.conf['output_path'] /
Matthieu Boileau's avatar
Matthieu Boileau committed
162
                              Path(path.name).with_suffix('.html'))
Matthieu Boileau's avatar
Matthieu Boileau committed
163
        return html_pages
164
165

    def build_pages(self):
Matthieu Boileau's avatar
Matthieu Boileau committed
166
167
168
        """
        Build a mini website using html templating and markdown conversion
        """
169

Matthieu Boileau's avatar
Matthieu Boileau committed
170
171
172
173
174
175
176
        # List mardkown files
        md_page_paths = list(self.conf['pages']['path'].glob('*.md'))
        # Instanciate all markdown pages to be rendered
        md_pages = [MarkdownPage(self, md_page_path)
                    for md_page_path in md_page_paths]

        def render_children(parent):
177
            """Recursive function to render children pages from parent page"""
Matthieu Boileau's avatar
Matthieu Boileau committed
178
179
180
181
182
183
            children = (md_page for md_page in md_pages
                        if md_page.parent_name == parent.name)
            for child in children:
                # Render markdown pages
                child.render(parent)
                render_children(child)
184
185

        # First build homepage
Matthieu Boileau's avatar
Matthieu Boileau committed
186
187
        homepage = HomePage(self)
        homepage.render()
188

189
        # Then build children pages
Matthieu Boileau's avatar
Matthieu Boileau committed
190
        render_children(homepage)
191

Matthieu Boileau's avatar
Matthieu Boileau committed
192
    def build_book(self):
193
        """Build a pdf book using bookbook"""
Matthieu Boileau's avatar
Matthieu Boileau committed
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240

        book = Book(self.conf)
        book.build()

    def task_output_dir(self):
        """Create empty output directory"""
        return {
            'targets': [self.conf['output_path']],
            'actions': [(create_folder, [self.conf['output_path']])],
            'uptodate': [self.conf['output_path'].is_dir],
            'clean': True,
        }

    def get_src_dst_paths(self, src_path: Path):
        """Return a tuple of file paths for source and destination"""
        files = get_file_list(src_path)
        src_files = [src_path / file for file in files]
        dst_path = self.conf['output_path'] / Path(src_path.name)
        dst_files = [dst_path / file for file in files]
        return src_files, dst_path, dst_files

    def task_copy_material(self):
        """Copy notebook and theme material to output directory"""
        for parent in 'nb', 'theme':
            for src_path in self.conf[parent]['material_paths']:
                src_files, dst_path, dst_files = self.get_src_dst_paths(
                    src_path)
                self.material += dst_files
                yield {
                    'name': dst_path,
                    'file_dep': src_files,
                    'targets': dst_files,
                    'actions': [(update_material, (src_path, dst_path))],
                    'clean': [(clean_tree, (dst_path,))],
                }

    def task_copy_reveal(self):
        """Copy reveal.js to output directory"""
        if self.conf['local_reveal']:
            src_path = self.conf['reveal_path']
            src_files, dst_path, dst_files = self.get_src_dst_paths(src_path)
            return {
                'file_dep': src_files,
                'targets': dst_files,
                'actions': [(update_material, (src_path, dst_path))],
                'clean': [(clean_tree, (dst_path,))],
            }
Matthieu Boileau's avatar
Matthieu Boileau committed
241
242
243
244
245
        else:
            return {
                'uptodate': [True],
                'actions': []
            }
Matthieu Boileau's avatar
Matthieu Boileau committed
246
247
248
249
250
251
252
253
254
255
256
257
258
259

    def execute_notebook(self, notebook: Path, executed_notebook: Path):
        """Execute notebook and write result to executed_notebook file"""
        with open(notebook) as f:
            nb = nbformat.read(f, as_version=4)

        ep = ExecutePreprocessor(timeout=60,
                                 kernel_name='python3',
                                 allow_errors=True)
        ep.preprocess(nb, {'metadata': {'path': self.conf['nb']['path']}})

        with open(executed_notebook, 'w', encoding='utf-8') as f:
            nbformat.write(nb, f)

Matthieu Boileau's avatar
Fix #15    
Matthieu Boileau committed
260
    def _get_executed_nb_path(self, notebook) -> Path:
Matthieu Boileau's avatar
Matthieu Boileau committed
261
262
263
264
265
266
        """Return executed notebook path"""
        return self.conf['output_path'] / Path(notebook.name)

    def task_execute_notebooks(self):
        """Write executed notebooks to output directory"""
        for notebook in self.notebooks:
Matthieu Boileau's avatar
Fix #15    
Matthieu Boileau committed
267
            executed_notebook = self._get_executed_nb_path(notebook)
Matthieu Boileau's avatar
Matthieu Boileau committed
268
269
270
            yield {
                'name': executed_notebook,
                'file_dep': [notebook],
Matthieu Boileau's avatar
Matthieu Boileau committed
271
                'task_dep': ['copy_material'],
Matthieu Boileau's avatar
Matthieu Boileau committed
272
273
274
275
276
277
278
279
                'targets': [executed_notebook],
                'clean': True,
                'actions': [(self.execute_notebook, (notebook,
                                                     executed_notebook))],
            }

    def convert_notebook(self, notebook: Path,
                         output_file: Path,
Matthieu Boileau's avatar
Matthieu Boileau committed
280
                         to='html'):
Matthieu Boileau's avatar
Matthieu Boileau committed
281
        """Convert notebook to output_file"""
Matthieu Boileau's avatar
Fix #15    
Matthieu Boileau committed
282
        log.debug(f"Converting {notebook} to {output_file}")
Matthieu Boileau's avatar
Matthieu Boileau committed
283
        if to == 'slide':
Matthieu Boileau's avatar
Matthieu Boileau committed
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
            exporter = self.slide_exporter
        else:
            exporter = self.html_exporter

        with open(notebook) as f:
            nb = nbformat.read(f, as_version=4)

        body, resources = exporter.from_notebook_node(nb)

        with open(output_file, 'w', encoding='utf-8') as f:
            f.write(body)

    def task_convert_to_html(self):
        """Convert executed notebook to html page"""
        for notebook in self.notebooks:
Matthieu Boileau's avatar
Fix #15    
Matthieu Boileau committed
299
            executed_notebook = self._get_executed_nb_path(notebook)
Matthieu Boileau's avatar
Matthieu Boileau committed
300
301
302
303
304
305
306
            html = executed_notebook.with_suffix('.html')
            self.htmls.append(html)
            yield {
                'name': html,
                'file_dep': [executed_notebook],
                'targets': [html],
                'clean': True,
Matthieu Boileau's avatar
Fix #15    
Matthieu Boileau committed
307
308
                'actions': [(self.convert_notebook,
                             (executed_notebook, html))],
Matthieu Boileau's avatar
Matthieu Boileau committed
309
310
311
312
313
            }

    def task_convert_to_slides(self):
        """Convert executed notebook to reveal slides"""
        for notebook in self.notebooks:
Matthieu Boileau's avatar
Fix #15    
Matthieu Boileau committed
314
            executed_notebook = self._get_executed_nb_path(notebook)
Matthieu Boileau's avatar
Matthieu Boileau committed
315
316
317
318
319
320
321
322
323
            slide = executed_notebook.with_suffix('.slides.html')
            self.slides.append(slide)
            yield {
                'name': slide,
                'file_dep': [executed_notebook],
                'uptodate': [config_changed(str(self.conf['local_reveal']))],
                'targets': [slide],
                'clean': True,
                'actions': [(self.convert_notebook,
Matthieu Boileau's avatar
Fix #15    
Matthieu Boileau committed
324
                             (executed_notebook, slide, 'slide'))],
Matthieu Boileau's avatar
Matthieu Boileau committed
325
326
327
            }

    def task_zip_chapters(self):
Matthieu Boileau's avatar
Matthieu Boileau committed
328
        """Build zip archives for single chapter downloads"""
Matthieu Boileau's avatar
Matthieu Boileau committed
329
330

        for notebook in self.notebooks:
Matthieu Boileau's avatar
Fix #15    
Matthieu Boileau committed
331
            zip_file = self._get_executed_nb_path(notebook).with_suffix('.zip')
Matthieu Boileau's avatar
Matthieu Boileau committed
332
333
334
335
336
337
338
            paths_to_zip = [notebook]
            for path in self.conf['nb']['material_paths']:
                paths_to_zip += get_file_list(path, relative=False)
            self.chapter_zips.append(zip_file.name)
            yield {
                'name': zip_file,
                'file_dep': paths_to_zip,
Matthieu Boileau's avatar
Matthieu Boileau committed
339
                'task_dep': ['copy_material'],
Matthieu Boileau's avatar
Matthieu Boileau committed
340
341
342
343
344
345
346
                'targets': [zip_file],
                'clean': True,
                'actions': [(zip_files, (zip_file, paths_to_zip))],
            }

    def task_build_pages(self):
        """Build html pages"""
347
        deps = self.executed_notebooks + self.md_page_paths + self.theme_files
Matthieu Boileau's avatar
Fix #16    
Matthieu Boileau committed
348
349
        if self.conf['config_file']:
            deps.append(self.conf['config_file'])
Matthieu Boileau's avatar
Matthieu Boileau committed
350
        return {
Matthieu Boileau's avatar
Matthieu Boileau committed
351
            'file_dep': deps,
Matthieu Boileau's avatar
Matthieu Boileau committed
352
            'task_dep': ['copy_material'],
Matthieu Boileau's avatar
Matthieu Boileau committed
353
354
355
356
357
358
359
            'targets': self.html_pages,
            'clean': True,
            'actions': [self.build_pages],
        }

    def task_build_book(self):
        """Build pdf book"""
Matthieu Boileau's avatar
Matthieu Boileau committed
360
361
362
363
364
365
366
367
368
369
370
371
        if self.book:
            return {
                'file_dep': self.executed_notebooks + [self.titlepage_path],
                'targets': [self.book],
                'clean': True,
                'actions': [self.build_book],
            }
        else:
            return {
                'uptodate': [True],
                'actions': []
            }
Matthieu Boileau's avatar
Matthieu Boileau committed
372
373
374

    def task_zip_archive(self):
        """Build a single zip archive for all material"""
Matthieu Boileau's avatar
Matthieu Boileau committed
375
        paths_to_zip = self.executed_notebooks + \
Matthieu Boileau's avatar
Matthieu Boileau committed
376
            self.html_pages + self.htmls + self.slides + self.material
Matthieu Boileau's avatar
Matthieu Boileau committed
377
378
        if self.book:
            paths_to_zip.append(self.book)
Matthieu Boileau's avatar
Matthieu Boileau committed
379
380
381
382
383
        return {
            'file_dep': paths_to_zip,
            'targets': [self.zip_file],
            'clean': True,
            'actions': [(zip_files, (self.zip_file, paths_to_zip),
Matthieu Boileau's avatar
Matthieu Boileau committed
384
385
                         {'ref_to_arc': (self.conf['output_path'],
                                         self.conf['slug_title'])})],
Matthieu Boileau's avatar
Matthieu Boileau committed
386
387
        }

Matthieu Boileau's avatar
Matthieu Boileau committed
388
389
390
391
    def build(self, args: list = None):
        """Build course using Doit"""
        if args is None:
            args = []
Matthieu Boileau's avatar
Matthieu Boileau committed
392
        doit_config = {}  # TODO: implement verbosity option
Matthieu Boileau's avatar
Matthieu Boileau committed
393
394
        return MyDoitMain(ClassTaskLoader(self),
                          extra_config=doit_config).run(args)