nbcourse.py 12.6 KB
Newer Older
Matthieu Boileau's avatar
Matthieu Boileau committed
1
2
#!/usr/bin/env python3
"""
Matthieu Boileau's avatar
Matthieu Boileau committed
3
4
5
Define a NbCourse object containing:
    - class methods for building html pages and pdf book
    - Doit tasks defined in task_*() functions
Matthieu Boileau's avatar
Matthieu Boileau committed
6
"""
Matthieu Boileau's avatar
Matthieu Boileau committed
7

8
from pathlib import Path
Matthieu Boileau's avatar
Matthieu Boileau committed
9
import yaml
Matthieu Boileau's avatar
Matthieu Boileau committed
10
import sys
Matthieu Boileau's avatar
Matthieu Boileau committed
11
from doit.tools import create_folder, config_changed
Matthieu Boileau's avatar
Matthieu Boileau committed
12
from .utils import update_material, clean_tree, get_file_list, update_dict, \
Matthieu Boileau's avatar
Matthieu Boileau committed
13
    zip_files
Matthieu Boileau's avatar
Matthieu Boileau committed
14
15
from .pages import HomePage, MarkdownPage
from .book import Book
16
import nbformat
Matthieu Boileau's avatar
Matthieu Boileau committed
17
18
19
from nbconvert.preprocessors import ExecutePreprocessor
from nbconvert import HTMLExporter, SlidesExporter
from .mydoit import MyDoitMain, ClassTaskLoader
20
import logging
21

22
23
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
24

25
DEFAULT_CONFIG_FILE = "nbcourse.yml"
Matthieu Boileau's avatar
Matthieu Boileau committed
26
27


28
29
class NbCourse:

Matthieu Boileau's avatar
Matthieu Boileau committed
30
31
32
33
34
35
36
37
38
39
40
    default_conf = {
                    'theme': {
                        'dir': 'theme/default',
                        'material': ('css', 'img')
                        },
                    'nb': {
                        'dir': 'notebooks',
                        'material': ('fig', 'exos')
                        },
                    'pages': {
                        'dir': 'pages',
Matthieu Boileau's avatar
Matthieu Boileau committed
41
                        'titlepage': 'titlepage.tex',
42
                        'home': 'index.html'
Matthieu Boileau's avatar
Matthieu Boileau committed
43
44
45
46
47
                        },
                    'bookbook_filename': 'bookbook.tex',
                    'local_reveal': False,
                    'slug_title': 'course',
                    'output_dir': 'build',
48
                    'title': 'A course',
Matthieu Boileau's avatar
Matthieu Boileau committed
49
50
51
52
53
54
55
                    }

    def __init__(self, config_file: Path = None):
        self.conf = self.default_conf.copy()
        if config_file:
            update_dict(self.conf, self._get_user_config(config_file))
            self.conf['project_path'] = Path(config_file).absolute().parent
Matthieu Boileau's avatar
Matthieu Boileau committed
56
57
58
        else:
            # Only default config is loaded (only useful for tests)
            self.conf['project_path'] = Path.cwd()
Matthieu Boileau's avatar
Matthieu Boileau committed
59
60
61
62
63
        self.conf['template_path'] = Path(self.conf['theme']['dir'],
                                          'templates')
        for key in 'nb', 'theme':
            self.conf[key]['path'] = Path(self.conf[key]['dir'])
            self.conf[key]['material_paths'] = [
Matthieu Boileau's avatar
Matthieu Boileau committed
64
65
                self.conf[key]['path'] / Path(d)
                for d in self.conf[key]['material']
Matthieu Boileau's avatar
Matthieu Boileau committed
66
67
68
69
70
71
72
73
74
75
                ]
        self.conf['pages']['path'] = Path(self.conf['pages']['dir'])
        self.conf['output_path'] = Path(self.conf['output_dir'])
        if self.conf['local_reveal']:
            libdir = Path(sys.argv[0]).parent
            self.conf['reveal_path'] = libdir / Path('reveal.js')
        self.conf['book_title'] = Path(self.conf['slug_title']).with_suffix(
            '.pdf')
        self.notebooks = tuple(self.conf['nb']['path'].glob('*-*.ipynb'))
        self.book = self.conf['output_path'] / self.conf['book_title']
76
        self.md_page_paths = list(self.conf['pages']['path'].glob('*.md'))
Matthieu Boileau's avatar
Matthieu Boileau committed
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
        self.html_pages = self._get_pages()  # homepage and documentation pages
        self.theme_files = [file for file in
                            self.conf['theme']['path'].glob('**/*')
                            if file.is_file()]
        self.htmls = []  # html export of notebooks
        self.slides = []  # reveal slide export of notebooks
        self.material = []  # additional material (pictures, etc.)
        self.chapter_zips = []

        self.executed_notebooks = [self.get_executed_notebook(notebook)
                                   for notebook in self.notebooks]
        self.zip_file = self.conf['output_path'] / \
            Path(self.conf['slug_title']).with_suffix('.zip')

        # Instantiate notebook exporter
        self.html_exporter = HTMLExporter()

        # Instantiate slide exporter
        self.slide_exporter = SlidesExporter()
        reveal_url = \
            "https://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.7.0"
        if self.conf['local_reveal']:
            self.slide_exporter.reveal_url_prefix = 'reveal.js'
        else:
            self.slide_exporter.reveal_url_prefix = reveal_url

    def _get_pages(self):
        """get a list of target html pages from markdown source"""
        # Start html_pages list with index.html
106
107
        html_pages = [self.conf['output_path'] / self.conf['pages']['home']]
        for path in self.md_page_paths:
Matthieu Boileau's avatar
Matthieu Boileau committed
108
            html_pages.append(self.conf['output_path'] /
109
                              Path(path.name).with_suffix('.html'))
Matthieu Boileau's avatar
Matthieu Boileau committed
110
        return html_pages
111
112

    @staticmethod
Matthieu Boileau's avatar
Matthieu Boileau committed
113
    def _get_user_config(config_file: Path) -> dict:
114
115
116
117
        """
        Parse yaml file to build the corresponding configuration dictionary
        """
        with open(config_file, 'r') as f:
Matthieu Boileau's avatar
Matthieu Boileau committed
118
            return yaml.safe_load(f)
Matthieu Boileau's avatar
Matthieu Boileau committed
119

120
    def build_pages(self):
Matthieu Boileau's avatar
Matthieu Boileau committed
121
122
123
        """
        Build a mini website using html templating and markdown conversion
        """
124

125
126
127
128
129
130
131
        # List mardkown files
        md_page_paths = list(self.conf['pages']['path'].glob('*.md'))
        # Instanciate all markdown pages to be rendered
        md_pages = [MarkdownPage(self, md_page_path)
                    for md_page_path in md_page_paths]

        def render_children(parent):
132
            """Recursive function to render children pages from parent page"""
133
134
135
136
137
138
            children = (md_page for md_page in md_pages
                        if md_page.parent_name == parent.name)
            for child in children:
                # Render markdown pages
                child.render(parent)
                render_children(child)
139
140

        # First build homepage
Matthieu Boileau's avatar
Matthieu Boileau committed
141
142
        homepage = HomePage(self)
        homepage.render()
143

144
        # Then build children pages
145
        render_children(homepage)
146

Matthieu Boileau's avatar
Matthieu Boileau committed
147
    def build_book(self):
148
        """Build a pdf book using bookbook"""
Matthieu Boileau's avatar
Matthieu Boileau committed
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195

        book = Book(self.conf)
        book.build()

    def task_output_dir(self):
        """Create empty output directory"""
        return {
            'targets': [self.conf['output_path']],
            'actions': [(create_folder, [self.conf['output_path']])],
            'uptodate': [self.conf['output_path'].is_dir],
            'clean': True,
        }

    def get_src_dst_paths(self, src_path: Path):
        """Return a tuple of file paths for source and destination"""
        files = get_file_list(src_path)
        src_files = [src_path / file for file in files]
        dst_path = self.conf['output_path'] / Path(src_path.name)
        dst_files = [dst_path / file for file in files]
        return src_files, dst_path, dst_files

    def task_copy_material(self):
        """Copy notebook and theme material to output directory"""
        for parent in 'nb', 'theme':
            for src_path in self.conf[parent]['material_paths']:
                src_files, dst_path, dst_files = self.get_src_dst_paths(
                    src_path)
                self.material += dst_files
                yield {
                    'name': dst_path,
                    'file_dep': src_files,
                    'targets': dst_files,
                    'actions': [(update_material, (src_path, dst_path))],
                    'clean': [(clean_tree, (dst_path,))],
                }

    def task_copy_reveal(self):
        """Copy reveal.js to output directory"""
        if self.conf['local_reveal']:
            src_path = self.conf['reveal_path']
            src_files, dst_path, dst_files = self.get_src_dst_paths(src_path)
            return {
                'file_dep': src_files,
                'targets': dst_files,
                'actions': [(update_material, (src_path, dst_path))],
                'clean': [(clean_tree, (dst_path,))],
            }
196
197
198
199
200
        else:
            return {
                'uptodate': [True],
                'actions': []
            }
Matthieu Boileau's avatar
Matthieu Boileau committed
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225

    def execute_notebook(self, notebook: Path, executed_notebook: Path):
        """Execute notebook and write result to executed_notebook file"""
        with open(notebook) as f:
            nb = nbformat.read(f, as_version=4)

        ep = ExecutePreprocessor(timeout=60,
                                 kernel_name='python3',
                                 allow_errors=True)
        ep.preprocess(nb, {'metadata': {'path': self.conf['nb']['path']}})

        with open(executed_notebook, 'w', encoding='utf-8') as f:
            nbformat.write(nb, f)

    def get_executed_notebook(self, notebook) -> Path:
        """Return executed notebook path"""
        return self.conf['output_path'] / Path(notebook.name)

    def task_execute_notebooks(self):
        """Write executed notebooks to output directory"""
        for notebook in self.notebooks:
            executed_notebook = self.get_executed_notebook(notebook)
            yield {
                'name': executed_notebook,
                'file_dep': [notebook],
226
                'task_dep': ['copy_material'],
Matthieu Boileau's avatar
Matthieu Boileau committed
227
228
229
230
231
232
233
234
                'targets': [executed_notebook],
                'clean': True,
                'actions': [(self.execute_notebook, (notebook,
                                                     executed_notebook))],
            }

    def convert_notebook(self, notebook: Path,
                         output_file: Path,
Matthieu Boileau's avatar
Matthieu Boileau committed
235
                         to='html'):
Matthieu Boileau's avatar
Matthieu Boileau committed
236
        """Convert notebook to output_file"""
Matthieu Boileau's avatar
Matthieu Boileau committed
237
        if to == 'slide':
Matthieu Boileau's avatar
Matthieu Boileau committed
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
            exporter = self.slide_exporter
        else:
            exporter = self.html_exporter

        with open(notebook) as f:
            nb = nbformat.read(f, as_version=4)

        body, resources = exporter.from_notebook_node(nb)

        with open(output_file, 'w', encoding='utf-8') as f:
            f.write(body)

    def task_convert_to_html(self):
        """Convert executed notebook to html page"""
        for notebook in self.notebooks:
            executed_notebook = self.get_executed_notebook(notebook)
            html = executed_notebook.with_suffix('.html')
            self.htmls.append(html)
            yield {
                'name': html,
                'file_dep': [executed_notebook],
                'targets': [html],
                'clean': True,
                'actions': [(self.convert_notebook, (notebook, html))],
            }

    def task_convert_to_slides(self):
        """Convert executed notebook to reveal slides"""
        for notebook in self.notebooks:
            executed_notebook = self.get_executed_notebook(notebook)
            slide = executed_notebook.with_suffix('.slides.html')
            self.slides.append(slide)
            yield {
                'name': slide,
                'file_dep': [executed_notebook],
                'uptodate': [config_changed(str(self.conf['local_reveal']))],
                'targets': [slide],
                'clean': True,
                'actions': [(self.convert_notebook,
                             (notebook, slide, 'slide'))],
            }

    def task_zip_chapters(self):
281
        """Build zip archives for single chapter downloads"""
Matthieu Boileau's avatar
Matthieu Boileau committed
282
283
284
285
286
287
288
289
290
291

        for notebook in self.notebooks:
            zip_file = self.get_executed_notebook(notebook).with_suffix('.zip')
            paths_to_zip = [notebook]
            for path in self.conf['nb']['material_paths']:
                paths_to_zip += get_file_list(path, relative=False)
            self.chapter_zips.append(zip_file.name)
            yield {
                'name': zip_file,
                'file_dep': paths_to_zip,
Matthieu Boileau's avatar
Matthieu Boileau committed
292
                'task_dep': ['copy_material'],
Matthieu Boileau's avatar
Matthieu Boileau committed
293
294
295
296
297
298
299
300
                'targets': [zip_file],
                'clean': True,
                'actions': [(zip_files, (zip_file, paths_to_zip))],
            }

    def task_build_pages(self):
        """Build html pages"""
        return {
301
            'file_dep': self.md_page_paths + self.theme_files,
302
            'task_dep': ['copy_material'],
Matthieu Boileau's avatar
Matthieu Boileau committed
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
            'targets': self.html_pages,
            'clean': True,
            'actions': [self.build_pages],
        }

    def task_build_book(self):
        """Build pdf book"""
        return {
            'file_dep': self.executed_notebooks,
            'targets': [self.book],
            'clean': True,
            'actions': [self.build_book],
        }

    def task_zip_archive(self):
        """Build a single zip archive for all material"""
        paths_to_zip = [self.book] + self.executed_notebooks + \
            self.html_pages + self.htmls + self.slides + self.material
        return {
            'file_dep': paths_to_zip,
            'targets': [self.zip_file],
            'clean': True,
            'actions': [(zip_files, (self.zip_file, paths_to_zip),
Matthieu Boileau's avatar
Matthieu Boileau committed
326
327
                         {'ref_to_arc': (self.conf['output_path'],
                                         self.conf['slug_title'])})],
Matthieu Boileau's avatar
Matthieu Boileau committed
328
329
        }

330
331
    def build(self, args: list = None):
        """Build course using Doit"""
332
333
        if args is None:
            args = []
Matthieu Boileau's avatar
Matthieu Boileau committed
334
335
336
        doit_config = {}  # TODO: implement verbosity option
        sys.exit(MyDoitMain(ClassTaskLoader(self),
                            extra_config=doit_config).run(args))