nbcourse.py 13.5 KB
Newer Older
Matthieu Boileau's avatar
Matthieu Boileau committed
1
2
#!/usr/bin/env python3
"""
Matthieu Boileau's avatar
Matthieu Boileau committed
3
4
5
Define a NbCourse object containing:
    - class methods for building html pages and pdf book
    - Doit tasks defined in task_*() functions
Matthieu Boileau's avatar
Matthieu Boileau committed
6
"""
Matthieu Boileau's avatar
Matthieu Boileau committed
7

8
from pathlib import Path
Matthieu Boileau's avatar
Matthieu Boileau committed
9
import yaml
Matthieu Boileau's avatar
Matthieu Boileau committed
10
import sys
Matthieu Boileau's avatar
Matthieu Boileau committed
11
from doit.tools import create_folder, config_changed
Matthieu Boileau's avatar
Matthieu Boileau committed
12
from .utils import update_material, clean_tree, get_file_list, update_dict, \
Matthieu Boileau's avatar
Matthieu Boileau committed
13
    zip_files
Matthieu Boileau's avatar
Matthieu Boileau committed
14
15
from .pages import HomePage, MarkdownPage
from .book import Book
16
import nbformat
Matthieu Boileau's avatar
Matthieu Boileau committed
17
18
19
from nbconvert.preprocessors import ExecutePreprocessor
from nbconvert import HTMLExporter, SlidesExporter
from .mydoit import MyDoitMain, ClassTaskLoader
Matthieu Boileau's avatar
Matthieu Boileau committed
20
import logging
21

Matthieu Boileau's avatar
Matthieu Boileau committed
22
23
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
24

25
DEFAULT_CONFIG_FILE = "nbcourse.yml"
Matthieu Boileau's avatar
Matthieu Boileau committed
26
27


28
29
class NbCourse:

Matthieu Boileau's avatar
Matthieu Boileau committed
30
31
32
33
34
35
36
37
38
39
40
    default_conf = {
                    'theme': {
                        'dir': 'theme/default',
                        'material': ('css', 'img')
                        },
                    'nb': {
                        'dir': 'notebooks',
                        'material': ('fig', 'exos')
                        },
                    'pages': {
                        'dir': 'pages',
Matthieu Boileau's avatar
Matthieu Boileau committed
41
                        'home': 'index.html'
Matthieu Boileau's avatar
Matthieu Boileau committed
42
                        },
Matthieu Boileau's avatar
Matthieu Boileau committed
43
44
45
46
                    'book': {
                        'titlepage': 'titlepage.tex',
                        'title': None,
                        },
Matthieu Boileau's avatar
Matthieu Boileau committed
47
48
49
                    'local_reveal': False,
                    'slug_title': 'course',
                    'output_dir': 'build',
Matthieu Boileau's avatar
Matthieu Boileau committed
50
51
52
53
54
55
56
57
58
                    'title': 'A course',
                    'links': [{
                        'title': 'Manual',
                        'target': 'manual.html',
                        'icon': {
                            'path': 'img/Infobox_info_icon.svg',
                            'width': '35px',
                            }
                        }]
Matthieu Boileau's avatar
Matthieu Boileau committed
59
60
61
                    }

    def __init__(self, config_file: Path = None):
Matthieu Boileau's avatar
Matthieu Boileau committed
62
        self.config_file = config_file
Matthieu Boileau's avatar
Matthieu Boileau committed
63
        self.conf = self.default_conf.copy()
Matthieu Boileau's avatar
Matthieu Boileau committed
64
65
66
        if self.config_file:
            update_dict(self.conf, self._get_user_config(self.config_file))
            self.conf['project_path'] = self.config_file.absolute().parent
Matthieu Boileau's avatar
Matthieu Boileau committed
67
68
69
        else:
            # Only default config is loaded (only useful for tests)
            self.conf['project_path'] = Path.cwd()
Matthieu Boileau's avatar
Matthieu Boileau committed
70
71
72
73
74
        self.conf['template_path'] = Path(self.conf['theme']['dir'],
                                          'templates')
        for key in 'nb', 'theme':
            self.conf[key]['path'] = Path(self.conf[key]['dir'])
            self.conf[key]['material_paths'] = [
Matthieu Boileau's avatar
Matthieu Boileau committed
75
76
                self.conf[key]['path'] / Path(d)
                for d in self.conf[key]['material']
Matthieu Boileau's avatar
Matthieu Boileau committed
77
78
79
80
81
82
83
                ]
        self.conf['pages']['path'] = Path(self.conf['pages']['dir'])
        self.conf['output_path'] = Path(self.conf['output_dir'])
        if self.conf['local_reveal']:
            libdir = Path(sys.argv[0]).parent
            self.conf['reveal_path'] = libdir / Path('reveal.js')
        self.notebooks = tuple(self.conf['nb']['path'].glob('*-*.ipynb'))
Matthieu Boileau's avatar
Matthieu Boileau committed
84
85
86
87
88
89
90
91

        if self.conf['book']['title']:
            self.titlepage_path = self.conf['pages']['path'] / \
                self.conf['book']['titlepage']
            self.book = self.conf['output_path'] / self.conf['book']['title']
        else:
            self.titlepage_path = None
            self.book = None
Matthieu Boileau's avatar
Matthieu Boileau committed
92
        self.md_page_paths = list(self.conf['pages']['path'].glob('*.md'))
Matthieu Boileau's avatar
Matthieu Boileau committed
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
        self.html_pages = self._get_pages()  # homepage and documentation pages
        self.theme_files = [file for file in
                            self.conf['theme']['path'].glob('**/*')
                            if file.is_file()]
        self.htmls = []  # html export of notebooks
        self.slides = []  # reveal slide export of notebooks
        self.material = []  # additional material (pictures, etc.)
        self.chapter_zips = []

        self.executed_notebooks = [self.get_executed_notebook(notebook)
                                   for notebook in self.notebooks]
        self.zip_file = self.conf['output_path'] / \
            Path(self.conf['slug_title']).with_suffix('.zip')

        # Instantiate notebook exporter
        self.html_exporter = HTMLExporter()

        # Instantiate slide exporter
        self.slide_exporter = SlidesExporter()
        reveal_url = \
            "https://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.7.0"
        if self.conf['local_reveal']:
            self.slide_exporter.reveal_url_prefix = 'reveal.js'
        else:
            self.slide_exporter.reveal_url_prefix = reveal_url

    def _get_pages(self):
        """get a list of target html pages from markdown source"""
        # Start html_pages list with index.html
Matthieu Boileau's avatar
Matthieu Boileau committed
122
123
        html_pages = [self.conf['output_path'] / self.conf['pages']['home']]
        for path in self.md_page_paths:
Matthieu Boileau's avatar
Matthieu Boileau committed
124
            html_pages.append(self.conf['output_path'] /
Matthieu Boileau's avatar
Matthieu Boileau committed
125
                              Path(path.name).with_suffix('.html'))
Matthieu Boileau's avatar
Matthieu Boileau committed
126
        return html_pages
127
128

    @staticmethod
Matthieu Boileau's avatar
Matthieu Boileau committed
129
    def _get_user_config(config_file: Path) -> dict:
130
131
132
133
        """
        Parse yaml file to build the corresponding configuration dictionary
        """
        with open(config_file, 'r') as f:
Matthieu Boileau's avatar
Matthieu Boileau committed
134
            return yaml.safe_load(f)
Matthieu Boileau's avatar
Matthieu Boileau committed
135

136
    def build_pages(self):
Matthieu Boileau's avatar
Matthieu Boileau committed
137
138
139
        """
        Build a mini website using html templating and markdown conversion
        """
140

Matthieu Boileau's avatar
Matthieu Boileau committed
141
142
143
144
145
146
147
        # List mardkown files
        md_page_paths = list(self.conf['pages']['path'].glob('*.md'))
        # Instanciate all markdown pages to be rendered
        md_pages = [MarkdownPage(self, md_page_path)
                    for md_page_path in md_page_paths]

        def render_children(parent):
148
            """Recursive function to render children pages from parent page"""
Matthieu Boileau's avatar
Matthieu Boileau committed
149
150
151
152
153
154
            children = (md_page for md_page in md_pages
                        if md_page.parent_name == parent.name)
            for child in children:
                # Render markdown pages
                child.render(parent)
                render_children(child)
155
156

        # First build homepage
Matthieu Boileau's avatar
Matthieu Boileau committed
157
158
        homepage = HomePage(self)
        homepage.render()
159

160
        # Then build children pages
Matthieu Boileau's avatar
Matthieu Boileau committed
161
        render_children(homepage)
162

Matthieu Boileau's avatar
Matthieu Boileau committed
163
    def build_book(self):
164
        """Build a pdf book using bookbook"""
Matthieu Boileau's avatar
Matthieu Boileau committed
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211

        book = Book(self.conf)
        book.build()

    def task_output_dir(self):
        """Create empty output directory"""
        return {
            'targets': [self.conf['output_path']],
            'actions': [(create_folder, [self.conf['output_path']])],
            'uptodate': [self.conf['output_path'].is_dir],
            'clean': True,
        }

    def get_src_dst_paths(self, src_path: Path):
        """Return a tuple of file paths for source and destination"""
        files = get_file_list(src_path)
        src_files = [src_path / file for file in files]
        dst_path = self.conf['output_path'] / Path(src_path.name)
        dst_files = [dst_path / file for file in files]
        return src_files, dst_path, dst_files

    def task_copy_material(self):
        """Copy notebook and theme material to output directory"""
        for parent in 'nb', 'theme':
            for src_path in self.conf[parent]['material_paths']:
                src_files, dst_path, dst_files = self.get_src_dst_paths(
                    src_path)
                self.material += dst_files
                yield {
                    'name': dst_path,
                    'file_dep': src_files,
                    'targets': dst_files,
                    'actions': [(update_material, (src_path, dst_path))],
                    'clean': [(clean_tree, (dst_path,))],
                }

    def task_copy_reveal(self):
        """Copy reveal.js to output directory"""
        if self.conf['local_reveal']:
            src_path = self.conf['reveal_path']
            src_files, dst_path, dst_files = self.get_src_dst_paths(src_path)
            return {
                'file_dep': src_files,
                'targets': dst_files,
                'actions': [(update_material, (src_path, dst_path))],
                'clean': [(clean_tree, (dst_path,))],
            }
Matthieu Boileau's avatar
Matthieu Boileau committed
212
213
214
215
216
        else:
            return {
                'uptodate': [True],
                'actions': []
            }
Matthieu Boileau's avatar
Matthieu Boileau committed
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241

    def execute_notebook(self, notebook: Path, executed_notebook: Path):
        """Execute notebook and write result to executed_notebook file"""
        with open(notebook) as f:
            nb = nbformat.read(f, as_version=4)

        ep = ExecutePreprocessor(timeout=60,
                                 kernel_name='python3',
                                 allow_errors=True)
        ep.preprocess(nb, {'metadata': {'path': self.conf['nb']['path']}})

        with open(executed_notebook, 'w', encoding='utf-8') as f:
            nbformat.write(nb, f)

    def get_executed_notebook(self, notebook) -> Path:
        """Return executed notebook path"""
        return self.conf['output_path'] / Path(notebook.name)

    def task_execute_notebooks(self):
        """Write executed notebooks to output directory"""
        for notebook in self.notebooks:
            executed_notebook = self.get_executed_notebook(notebook)
            yield {
                'name': executed_notebook,
                'file_dep': [notebook],
Matthieu Boileau's avatar
Matthieu Boileau committed
242
                'task_dep': ['copy_material'],
Matthieu Boileau's avatar
Matthieu Boileau committed
243
244
245
246
247
248
249
250
                'targets': [executed_notebook],
                'clean': True,
                'actions': [(self.execute_notebook, (notebook,
                                                     executed_notebook))],
            }

    def convert_notebook(self, notebook: Path,
                         output_file: Path,
Matthieu Boileau's avatar
Matthieu Boileau committed
251
                         to='html'):
Matthieu Boileau's avatar
Matthieu Boileau committed
252
        """Convert notebook to output_file"""
Matthieu Boileau's avatar
Matthieu Boileau committed
253
        if to == 'slide':
Matthieu Boileau's avatar
Matthieu Boileau committed
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
            exporter = self.slide_exporter
        else:
            exporter = self.html_exporter

        with open(notebook) as f:
            nb = nbformat.read(f, as_version=4)

        body, resources = exporter.from_notebook_node(nb)

        with open(output_file, 'w', encoding='utf-8') as f:
            f.write(body)

    def task_convert_to_html(self):
        """Convert executed notebook to html page"""
        for notebook in self.notebooks:
            executed_notebook = self.get_executed_notebook(notebook)
            html = executed_notebook.with_suffix('.html')
            self.htmls.append(html)
            yield {
                'name': html,
                'file_dep': [executed_notebook],
                'targets': [html],
                'clean': True,
                'actions': [(self.convert_notebook, (notebook, html))],
            }

    def task_convert_to_slides(self):
        """Convert executed notebook to reveal slides"""
        for notebook in self.notebooks:
            executed_notebook = self.get_executed_notebook(notebook)
            slide = executed_notebook.with_suffix('.slides.html')
            self.slides.append(slide)
            yield {
                'name': slide,
                'file_dep': [executed_notebook],
                'uptodate': [config_changed(str(self.conf['local_reveal']))],
                'targets': [slide],
                'clean': True,
                'actions': [(self.convert_notebook,
                             (notebook, slide, 'slide'))],
            }

    def task_zip_chapters(self):
Matthieu Boileau's avatar
Matthieu Boileau committed
297
        """Build zip archives for single chapter downloads"""
Matthieu Boileau's avatar
Matthieu Boileau committed
298
299
300
301
302
303
304
305
306
307

        for notebook in self.notebooks:
            zip_file = self.get_executed_notebook(notebook).with_suffix('.zip')
            paths_to_zip = [notebook]
            for path in self.conf['nb']['material_paths']:
                paths_to_zip += get_file_list(path, relative=False)
            self.chapter_zips.append(zip_file.name)
            yield {
                'name': zip_file,
                'file_dep': paths_to_zip,
Matthieu Boileau's avatar
Matthieu Boileau committed
308
                'task_dep': ['copy_material'],
Matthieu Boileau's avatar
Matthieu Boileau committed
309
310
311
312
313
314
315
                'targets': [zip_file],
                'clean': True,
                'actions': [(zip_files, (zip_file, paths_to_zip))],
            }

    def task_build_pages(self):
        """Build html pages"""
316
        deps = self.executed_notebooks + self.md_page_paths + self.theme_files
Matthieu Boileau's avatar
Matthieu Boileau committed
317
318
        if self.config_file:
            deps.append(self.config_file)
Matthieu Boileau's avatar
Matthieu Boileau committed
319
        return {
Matthieu Boileau's avatar
Matthieu Boileau committed
320
            'file_dep': deps,
Matthieu Boileau's avatar
Matthieu Boileau committed
321
            'task_dep': ['copy_material'],
Matthieu Boileau's avatar
Matthieu Boileau committed
322
323
324
325
326
327
328
            'targets': self.html_pages,
            'clean': True,
            'actions': [self.build_pages],
        }

    def task_build_book(self):
        """Build pdf book"""
Matthieu Boileau's avatar
Matthieu Boileau committed
329
330
331
332
333
334
335
336
337
338
339
340
        if self.book:
            return {
                'file_dep': self.executed_notebooks + [self.titlepage_path],
                'targets': [self.book],
                'clean': True,
                'actions': [self.build_book],
            }
        else:
            return {
                'uptodate': [True],
                'actions': []
            }
Matthieu Boileau's avatar
Matthieu Boileau committed
341
342
343

    def task_zip_archive(self):
        """Build a single zip archive for all material"""
Matthieu Boileau's avatar
Matthieu Boileau committed
344
        paths_to_zip = self.executed_notebooks + \
Matthieu Boileau's avatar
Matthieu Boileau committed
345
            self.html_pages + self.htmls + self.slides + self.material
Matthieu Boileau's avatar
Matthieu Boileau committed
346
347
        if self.book:
            paths_to_zip.append(self.book)
Matthieu Boileau's avatar
Matthieu Boileau committed
348
349
350
351
352
        return {
            'file_dep': paths_to_zip,
            'targets': [self.zip_file],
            'clean': True,
            'actions': [(zip_files, (self.zip_file, paths_to_zip),
Matthieu Boileau's avatar
Matthieu Boileau committed
353
354
                         {'ref_to_arc': (self.conf['output_path'],
                                         self.conf['slug_title'])})],
Matthieu Boileau's avatar
Matthieu Boileau committed
355
356
        }

Matthieu Boileau's avatar
Matthieu Boileau committed
357
358
359
360
    def build(self, args: list = None):
        """Build course using Doit"""
        if args is None:
            args = []
Matthieu Boileau's avatar
Matthieu Boileau committed
361
362
363
        doit_config = {}  # TODO: implement verbosity option
        sys.exit(MyDoitMain(ClassTaskLoader(self),
                            extra_config=doit_config).run(args))