nbcourse.py 12.4 KB
Newer Older
Matthieu Boileau's avatar
Matthieu Boileau committed
1
2
#!/usr/bin/env python3
"""
Matthieu Boileau's avatar
Matthieu Boileau committed
3
4
5
Define a NbCourse object containing:
    - class methods for building html pages and pdf book
    - Doit tasks defined in task_*() functions
Matthieu Boileau's avatar
Matthieu Boileau committed
6
"""
Matthieu Boileau's avatar
Matthieu Boileau committed
7

8
from pathlib import Path
Matthieu Boileau's avatar
Matthieu Boileau committed
9
import yaml
Matthieu Boileau's avatar
Matthieu Boileau committed
10
import sys
Matthieu Boileau's avatar
Matthieu Boileau committed
11
from doit.tools import create_folder, config_changed
Matthieu Boileau's avatar
Matthieu Boileau committed
12
from .utils import update_material, clean_tree, get_file_list, update_dict, \
Matthieu Boileau's avatar
Matthieu Boileau committed
13
    zip_files
Matthieu Boileau's avatar
Matthieu Boileau committed
14
15
from .pages import HomePage, MarkdownPage
from .book import Book
16
import nbformat
Matthieu Boileau's avatar
Matthieu Boileau committed
17
18
19
from nbconvert.preprocessors import ExecutePreprocessor
from nbconvert import HTMLExporter, SlidesExporter
from .mydoit import MyDoitMain, ClassTaskLoader
20

21

22
DEFAULT_CONFIG_FILE = "nbcourse.yml"
Matthieu Boileau's avatar
Matthieu Boileau committed
23
24


25
26
class NbCourse:

Matthieu Boileau's avatar
Matthieu Boileau committed
27
28
29
30
31
32
33
34
35
36
37
    default_conf = {
                    'theme': {
                        'dir': 'theme/default',
                        'material': ('css', 'img')
                        },
                    'nb': {
                        'dir': 'notebooks',
                        'material': ('fig', 'exos')
                        },
                    'pages': {
                        'dir': 'pages',
Matthieu Boileau's avatar
Matthieu Boileau committed
38
                        'titlepage': 'titlepage.tex',
Matthieu Boileau's avatar
Matthieu Boileau committed
39
40
41
42
43
44
                        'home': {'name': 'index.html'}
                        },
                    'bookbook_filename': 'bookbook.tex',
                    'local_reveal': False,
                    'slug_title': 'course',
                    'output_dir': 'build',
45
                    'title': 'A course'
Matthieu Boileau's avatar
Matthieu Boileau committed
46
47
48
49
50
51
52
                    }

    def __init__(self, config_file: Path = None):
        self.conf = self.default_conf.copy()
        if config_file:
            update_dict(self.conf, self._get_user_config(config_file))
            self.conf['project_path'] = Path(config_file).absolute().parent
Matthieu Boileau's avatar
Matthieu Boileau committed
53
54
55
        else:
            # Only default config is loaded (only useful for tests)
            self.conf['project_path'] = Path.cwd()
Matthieu Boileau's avatar
Matthieu Boileau committed
56
57
58
59
60
        self.conf['template_path'] = Path(self.conf['theme']['dir'],
                                          'templates')
        for key in 'nb', 'theme':
            self.conf[key]['path'] = Path(self.conf[key]['dir'])
            self.conf[key]['material_paths'] = [
Matthieu Boileau's avatar
Matthieu Boileau committed
61
62
                self.conf[key]['path'] / Path(d)
                for d in self.conf[key]['material']
Matthieu Boileau's avatar
Matthieu Boileau committed
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
                ]
        self.conf['pages']['path'] = Path(self.conf['pages']['dir'])
        self.conf['output_path'] = Path(self.conf['output_dir'])
        if self.conf['local_reveal']:
            libdir = Path(sys.argv[0]).parent
            self.conf['reveal_path'] = libdir / Path('reveal.js')
        self.conf['book_title'] = Path(self.conf['slug_title']).with_suffix(
            '.pdf')
        self.notebooks = tuple(self.conf['nb']['path'].glob('*-*.ipynb'))
        self.book = self.conf['output_path'] / self.conf['book_title']
        self.md_pages = list(self.conf['pages']['path'].glob('*.md'))
        self.html_pages = self._get_pages()  # homepage and documentation pages
        self.theme_files = [file for file in
                            self.conf['theme']['path'].glob('**/*')
                            if file.is_file()]
        self.htmls = []  # html export of notebooks
        self.slides = []  # reveal slide export of notebooks
        self.material = []  # additional material (pictures, etc.)
        self.chapter_zips = []

        self.executed_notebooks = [self.get_executed_notebook(notebook)
                                   for notebook in self.notebooks]
        self.zip_file = self.conf['output_path'] / \
            Path(self.conf['slug_title']).with_suffix('.zip')

        # Instantiate notebook exporter
        self.html_exporter = HTMLExporter()

        # Instantiate slide exporter
        self.slide_exporter = SlidesExporter()
        reveal_url = \
            "https://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.7.0"
        if self.conf['local_reveal']:
            self.slide_exporter.reveal_url_prefix = 'reveal.js'
        else:
            self.slide_exporter.reveal_url_prefix = reveal_url

    def _get_pages(self):
        """get a list of target html pages from markdown source"""
        # Start html_pages list with index.html
        html_pages = [self.conf['output_path'] /
                      self.conf['pages']['home']['name']]
        for md_page in self.md_pages:
            html_pages.append(self.conf['output_path'] /
                              md_page.relative_to(
                              self.conf['pages']['path']).with_suffix('.html'))
        return html_pages
110
111

    @staticmethod
Matthieu Boileau's avatar
Matthieu Boileau committed
112
    def _get_user_config(config_file: Path) -> dict:
113
114
115
116
        """
        Parse yaml file to build the corresponding configuration dictionary
        """
        with open(config_file, 'r') as f:
Matthieu Boileau's avatar
Matthieu Boileau committed
117
            return yaml.safe_load(f)
Matthieu Boileau's avatar
Matthieu Boileau committed
118

119
    def build_pages(self):
Matthieu Boileau's avatar
Matthieu Boileau committed
120
121
122
        """
        Build a mini website using html templating and markdown conversion
        """
123
124
125
126
127
128
129

        def render_children(parent, parent_page):
            """Recursive function to render children pages from parent page"""
            try:
                children = parent['children']
                for child in children.values():
                    # Render markdown pages
Matthieu Boileau's avatar
Matthieu Boileau committed
130
                    child_page = MarkdownPage(self, child['html'],
131
132
133
                                              title=child['title'],
                                              src=child['md'],
                                              parent=parent_page)
Matthieu Boileau's avatar
Matthieu Boileau committed
134
                    child_page.render()
135
136
137
138
139
140
                    render_children(child, child_page)
            except KeyError:
                # Parent page has no children
                return

        # First build homepage
Matthieu Boileau's avatar
Matthieu Boileau committed
141
142
        homepage = HomePage(self)
        homepage.render()
143

144
        # Then build children pages
Matthieu Boileau's avatar
Matthieu Boileau committed
145
        render_children(self.conf['pages']['home'], homepage)
146

Matthieu Boileau's avatar
Matthieu Boileau committed
147
    def build_book(self):
148
        """Build a pdf book using bookbook"""
Matthieu Boileau's avatar
Matthieu Boileau committed
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228

        book = Book(self.conf)
        book.build()

    def task_output_dir(self):
        """Create empty output directory"""
        return {
            'targets': [self.conf['output_path']],
            'actions': [(create_folder, [self.conf['output_path']])],
            'uptodate': [self.conf['output_path'].is_dir],
            'clean': True,
        }

    def get_src_dst_paths(self, src_path: Path):
        """Return a tuple of file paths for source and destination"""
        files = get_file_list(src_path)
        src_files = [src_path / file for file in files]
        dst_path = self.conf['output_path'] / Path(src_path.name)
        dst_files = [dst_path / file for file in files]
        return src_files, dst_path, dst_files

    def task_copy_material(self):
        """Copy notebook and theme material to output directory"""
        for parent in 'nb', 'theme':
            for src_path in self.conf[parent]['material_paths']:
                src_files, dst_path, dst_files = self.get_src_dst_paths(
                    src_path)
                self.material += dst_files
                yield {
                    'name': dst_path,
                    'file_dep': src_files,
                    'targets': dst_files,
                    'actions': [(update_material, (src_path, dst_path))],
                    'clean': [(clean_tree, (dst_path,))],
                }

    def task_copy_reveal(self):
        """Copy reveal.js to output directory"""
        if self.conf['local_reveal']:
            src_path = self.conf['reveal_path']
            src_files, dst_path, dst_files = self.get_src_dst_paths(src_path)
            return {
                'file_dep': src_files,
                'targets': dst_files,
                'actions': [(update_material, (src_path, dst_path))],
                'clean': [(clean_tree, (dst_path,))],
            }

    def execute_notebook(self, notebook: Path, executed_notebook: Path):
        """Execute notebook and write result to executed_notebook file"""
        with open(notebook) as f:
            nb = nbformat.read(f, as_version=4)

        ep = ExecutePreprocessor(timeout=60,
                                 kernel_name='python3',
                                 allow_errors=True)
        ep.preprocess(nb, {'metadata': {'path': self.conf['nb']['path']}})

        with open(executed_notebook, 'w', encoding='utf-8') as f:
            nbformat.write(nb, f)

    def get_executed_notebook(self, notebook) -> Path:
        """Return executed notebook path"""
        return self.conf['output_path'] / Path(notebook.name)

    def task_execute_notebooks(self):
        """Write executed notebooks to output directory"""
        for notebook in self.notebooks:
            executed_notebook = self.get_executed_notebook(notebook)
            yield {
                'name': executed_notebook,
                'file_dep': [notebook],
                'targets': [executed_notebook],
                'clean': True,
                'actions': [(self.execute_notebook, (notebook,
                                                     executed_notebook))],
            }

    def convert_notebook(self, notebook: Path,
                         output_file: Path,
Matthieu Boileau's avatar
Matthieu Boileau committed
229
                         to='html'):
Matthieu Boileau's avatar
Matthieu Boileau committed
230
        """Convert notebook to output_file"""
Matthieu Boileau's avatar
Matthieu Boileau committed
231
        if to == 'slide':
Matthieu Boileau's avatar
Matthieu Boileau committed
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
            exporter = self.slide_exporter
        else:
            exporter = self.html_exporter

        with open(notebook) as f:
            nb = nbformat.read(f, as_version=4)

        body, resources = exporter.from_notebook_node(nb)

        with open(output_file, 'w', encoding='utf-8') as f:
            f.write(body)

    def task_convert_to_html(self):
        """Convert executed notebook to html page"""
        for notebook in self.notebooks:
            executed_notebook = self.get_executed_notebook(notebook)
            html = executed_notebook.with_suffix('.html')
            self.htmls.append(html)
            yield {
                'name': html,
                'file_dep': [executed_notebook],
                'targets': [html],
                'clean': True,
                'actions': [(self.convert_notebook, (notebook, html))],
            }

    def task_convert_to_slides(self):
        """Convert executed notebook to reveal slides"""
        for notebook in self.notebooks:
            executed_notebook = self.get_executed_notebook(notebook)
            slide = executed_notebook.with_suffix('.slides.html')
            self.slides.append(slide)
            yield {
                'name': slide,
                'file_dep': [executed_notebook],
                'uptodate': [config_changed(str(self.conf['local_reveal']))],
                'targets': [slide],
                'clean': True,
                'actions': [(self.convert_notebook,
                             (notebook, slide, 'slide'))],
            }

    def task_zip_chapters(self):
        """Build one zip archive per chapter"""

        for notebook in self.notebooks:
            zip_file = self.get_executed_notebook(notebook).with_suffix('.zip')
            paths_to_zip = [notebook]
            for path in self.conf['nb']['material_paths']:
                paths_to_zip += get_file_list(path, relative=False)
            self.chapter_zips.append(zip_file.name)
            yield {
                'name': zip_file,
                'file_dep': paths_to_zip,
                'targets': [zip_file],
                'clean': True,
                'actions': [(zip_files, (zip_file, paths_to_zip))],
            }

    def task_build_pages(self):
        """Build html pages"""
        return {
            'file_dep': self.md_pages + self.theme_files,
            'targets': self.html_pages,
            'clean': True,
            'actions': [self.build_pages],
        }

    def task_build_book(self):
        """Build pdf book"""
        return {
            'file_dep': self.executed_notebooks,
            'targets': [self.book],
            'clean': True,
            'actions': [self.build_book],
        }

    def task_zip_archive(self):
        """Build a single zip archive for all material"""
        paths_to_zip = [self.book] + self.executed_notebooks + \
            self.html_pages + self.htmls + self.slides + self.material
        return {
            'file_dep': paths_to_zip,
            'targets': [self.zip_file],
            'clean': True,
            'actions': [(zip_files, (self.zip_file, paths_to_zip),
Matthieu Boileau's avatar
Matthieu Boileau committed
318
319
                         {'ref_to_arc': (self.conf['output_path'],
                                         self.conf['slug_title'])})],
Matthieu Boileau's avatar
Matthieu Boileau committed
320
321
        }

322
    def build(self, args: list = []):
Matthieu Boileau's avatar
Matthieu Boileau committed
323
324
325
        doit_config = {}  # TODO: implement verbosity option
        sys.exit(MyDoitMain(ClassTaskLoader(self),
                            extra_config=doit_config).run(args))