Commit d87036f3 authored by Matthieu Boileau's avatar Matthieu Boileau
Browse files

Merge branch 'dev-doit' into 'master'

Dev doit

See merge request !1
parents 90cf5c6a 9dcd93ce
......@@ -5,6 +5,6 @@
__pycache__/
.idea/
.vscode/
venv/
.env/
build/
notebooks/
\ No newline at end of file
notebooks/
[submodule "skeleton/reveal.js"]
path = skeleton/reveal.js
[submodule "reveal.js"]
path = reveal.js
url = https://github.com/hakimel/reveal.js.git
branch = 3.7.0
......@@ -6,24 +6,37 @@ import argparse
from .nbcourse import NbCourse, DEFAULT_CONFIG_FILE
from .quickstart import initialize
from pathlib import Path
from .mydoit import MyDoit, ClassTaskLoader
import sys
def main():
parser = argparse.ArgumentParser(description=__name__.__doc__)
course = NbCourse()
loader = ClassTaskLoader(course)
prog = globals()['__package__']
description = globals()['__doc__']
formatter = argparse.RawDescriptionHelpFormatter
epilog = MyDoit(loader).get_help()
parser = argparse.ArgumentParser(prog=prog,
description=description,
formatter_class=formatter,
epilog=epilog)
init = parser.add_argument_group('initialize')
init.add_argument('--init', action='store_true',
help=initialize.__doc__)
generate = parser.add_argument_group('generate')
generate.add_argument('--config', type=Path, default=DEFAULT_CONFIG_FILE,
help='load YAML file containing site configuration')
generate.add_argument('--book', type=Path,
help='Generate a pdf book using bookbook')
parser.add_argument('--init', type=str, dest='course_title',
help='initiate a nbcourse directory')
args = parser.parse_args()
if args.course_title:
initialize(args.course_title)
generate.add_argument('--config', dest='config_file', type=Path,
default=DEFAULT_CONFIG_FILE,
help='Lad YAML file describing site configuration '
'(default: %(default)s)')
# Handle undefined arguments
parsed, extra = parser.parse_known_args()
if parsed.init:
initialize()
else:
course = NbCourse(args.config)
if args.book:
course.build_book(args.book)
else:
course.build_pages()
course = NbCourse(parsed.config_file)
course.run(extra)
import jinja2
import logging
import os
from bookbook.latex import combine_and_convert
from pathlib import Path
logging.basicConfig(level=logging.INFO)
log = logging.getLogger(__name__)
class BookContext:
"""
A context manager to work in a directory with a temporary file
"""
def __init__(self, filename, filecontent: str, work_path: Path):
self.filepath = Path(filename)
self.filecontent = filecontent
self.work_path = work_path
self.curdir = None
def __enter__(self):
self.initial_path = Path.cwd()
os.chdir(self.work_path)
with open(self.filepath, 'w') as f:
f.write(self.filecontent)
return self
def __exit__(self, *args):
os.remove(self.filepath)
os.chdir(self.initial_path)
class Book:
"""A book to be built with bookbook"""
def __init__(self, conf):
self.output_path = conf['output_path']
self.project_path = conf['project_path']
self.titlepage_path = conf['pages']['path'] / Path("titlepage.tex")
self.book_title = conf['book_title']
self.bookbook_filename = conf['bookbook_filename']
self.template_path = conf['template_path'].as_posix()
def build(self):
"""Build book"""
# Render basic bookbook template including /pages/titlepage.tex content
env = jinja2.Environment(loader=jinja2.FileSystemLoader(
self.template_path))
template = env.get_template('book.tplx')
with open(self.titlepage_path) as f:
titlepage_content = f.read()
bookbook_template = template.render(titlepage=titlepage_content)
# Work temporarly in self.output_path directory
with BookContext(Path(self.bookbook_filename), bookbook_template,
self.output_path):
# Call bookbook
combine_and_convert(Path('.'),
Path(self.book_title),
pdf=True,
template_file=self.bookbook_filename)
"""Define some Doit customizations"""
from doit.cmd_help import Help
from doit.doit_cmd import DoitMain
from doit.cmd_base import TaskLoader, ModuleTaskLoader
import inspect
class MyDoitMain(DoitMain):
BIN_NAME = globals()['__package__']
class MyHelp(Help):
"""Extend doit Help class to add a get_usage() function"""
def get_usage(self, cmds):
"""return doit "usage" (basic help) instructions"""
msg = "\n"
for cmd_name in sorted(cmds.keys()):
cmd = cmds[cmd_name]
msg += " {:16s} {}\n".format(cmd_name, cmd.doc_purpose)
msg += "\n"
for line in [
" {} show doit's help\n",
" {} task show help on task dictionary fields\n",
" {} <command> show command usage\n",
" {} <task-name> show task usage\n"]:
msg += line.format("help")
return msg
class MyDoit(DoitMain):
"""Extend DoitMain class to add a get_help() function"""
def get_help(self):
"""return help usage as string"""
# get list of available commands
sub_cmds = self.get_cmds()
help = MyHelp(config=self.config,
bin_name='',
cmds=sub_cmds)
return help.get_usage(sub_cmds.to_dict())
class ClassTaskLoader(ModuleTaskLoader):
"""Implementation of a loader of tasks from a Class namespace"""
def load_tasks(self, cmd, params, args):
# Build dict from class namespace
self.mod_dict = dict(inspect.getmembers(self.mod_dict))
return self._load_from(cmd, self.mod_dict, self.cmd_names)
#!/usr/bin/env python3
"""
Build a small website to host Jupyter notebooks as course chapters
Define a NbCourse object containing:
- class methods for building html pages and pdf book
- Doit tasks defined in task_*() functions
"""
import os
from pathlib import Path
import jinja2
import yaml
from pprint import pformat
import markdown
from bs4 import BeautifulSoup
import re
import sys
from doit.tools import result_dep, create_folder, config_changed
from .utils import update_material, clean_tree, get_file_list, update_dict, \
zip_files, clean_file
from .pages import HomePage, MarkdownPage
from .book import Book
import nbformat
import logging
from bookbook.latex import combine_and_convert
from nbconvert.preprocessors import ExecutePreprocessor
from nbconvert import HTMLExporter, SlidesExporter
from .mydoit import MyDoitMain, ClassTaskLoader
DEFAULT_CONFIG_FILE = "nbcourse.yml"
THEME_DIR = 'theme/default'
TEMPLATE_PATH = Path(THEME_DIR, 'templates')
NB_DIR = 'notebooks'
DEFAULT_PDF_TITLE = 'nbcourse.pdf'
BOOKBOOK_FILENAME = 'bookbook.tex'
log = logging.getLogger(__name__)
class HomePage:
"""A class to handle a homepage to be rendered"""
html_template = "index.html"
def __init__(self, html):
self.html = html
self.title = None
self.parent = None
self.variables = {}
self.chapters = {}
def _get_chapter(self, path: Path):
"""Get chapter from notebook file (source: bookbook)"""
chapter_no = int(re.match(r'(\d+)\-', path.stem).group(1))
nb = nbformat.read(str(path), as_version=4)
assert nb.cells[0].cell_type == 'markdown', nb.cells[0].cell_type
lines = nb.cells[0].source.splitlines()
if lines[0].startswith('# '):
header = lines[0][2:]
elif len(lines) > 1 and lines[1].startswith('==='):
header = lines[0]
else:
assert False, "No heading found in {}".format(path)
assert path.suffix == '.ipynb', path
return {'number': chapter_no,
'title': header,
'filename': path.stem}
def _get_variables(self, config: dict):
"""Set some variables from config dictionnary and notebooks files"""
logging.basicConfig(level=logging.INFO)
self.title = config['title']
self.nbdir = config['configpath'] / Path(config.get('nbdir', NB_DIR))
self.variables = config
# Get chapters from notebook files
chapters = []
for nbfile in self.nbdir.glob('*-*.ipynb'):
chapter = self._get_chapter(nbfile)
chapter['preview_only'] = True if chapter['number'] \
in config['chapter_preview_only'] else False
chapters.append(chapter)
chapters.sort(key=lambda chapter: chapter['number'])
self.variables.update({'chapters': chapters})
log.info("Homepage template variables:")
log.info(pformat(self.variables))
def _render_template(self, config):
"""Return html rendered from template and variables"""
# Inject variables into html_template
full_template_path = config['configpath'] / TEMPLATE_PATH
env = jinja2.Environment(loader=jinja2.FileSystemLoader(
full_template_path.as_posix()))
template = env.get_template(self.html_template)
html_out = template.render(self.variables)
with open(self.html, 'w') as f:
f.write(html_out)
def render(self, config):
"""Render html using templating"""
self._get_variables(config)
self._render_template(config)
class MarkdownPage(HomePage):
html_template = "article.html"
pages_path = Path('pages')
def __init__(self, html, title, src, parent=None):
super().__init__(html)
self.title = title
self.src = Path(src)
self.parent = parent
def render(self, config):
"""Render markdown file into html file adding ToC"""
pattern = re.compile(r'^doc\/(.*)\.md$')
def markdown2html():
"""Return html from markdown file"""
with open(config['configpath'] / self.pages_path /
self.src, 'r') as f:
text = f.read()
md = markdown.Markdown(extensions=['fenced_code', 'codehilite',
'toc'])
body = md.convert(text)
# Transform link to file.md into link to file.html
soup = BeautifulSoup(body, 'html.parser')
for link in soup.findAll(['a']):
href = link.get('href')
link['href'] = pattern.sub(r'\1.html', href)
return soup.prettify(), md.toc
def get_menu_list():
"""Return a list to be displayed as a top menu"""
# At least current page
menu = [(self.title, self.html)]
page = self
while page.parent.parent:
menu.append((page.parent.title, page.parent.html))
page = page.parent
menu.reverse()
return menu
body, toc = markdown2html()
self.variables = {'title': self.title,
'article_content': body,
'toc': toc,
'menu': get_menu_list()}
self._render_template(config)
class NbCourse:
def __init__(self, config_file=DEFAULT_CONFIG_FILE):
self.config = self._get_config(config_file)
default_conf = {
'theme': {
'dir': 'theme/default',
'material': ('css', 'img')
},
'nb': {
'dir': 'notebooks',
'material': ('fig', 'exos')
},
'pages': {
'dir': 'pages',
'home': {'name': 'index.html'}
},
'bookbook_filename': 'bookbook.tex',
'local_reveal': False,
'slug_title': 'course',
'output_dir': 'build',
}
def __init__(self, config_file: Path = None):
self.conf = self.default_conf.copy()
if config_file:
update_dict(self.conf, self._get_user_config(config_file))
self.conf['project_path'] = Path(config_file).absolute().parent
self.conf['template_path'] = Path(self.conf['theme']['dir'],
'templates')
for key in 'nb', 'theme':
self.conf[key]['path'] = Path(self.conf[key]['dir'])
self.conf[key]['material_paths'] = [
self.conf[key]['path'] / Path(dir)
for dir in self.conf[key]['material']
]
self.conf['pages']['path'] = Path(self.conf['pages']['dir'])
self.conf['output_path'] = Path(self.conf['output_dir'])
if self.conf['local_reveal']:
libdir = Path(sys.argv[0]).parent
self.conf['reveal_path'] = libdir / Path('reveal.js')
self.conf['book_title'] = Path(self.conf['slug_title']).with_suffix(
'.pdf')
self.notebooks = tuple(self.conf['nb']['path'].glob('*-*.ipynb'))
self.book = self.conf['output_path'] / self.conf['book_title']
self.md_pages = list(self.conf['pages']['path'].glob('*.md'))
self.html_pages = self._get_pages() # homepage and documentation pages
self.theme_files = [file for file in
self.conf['theme']['path'].glob('**/*')
if file.is_file()]
self.htmls = [] # html export of notebooks
self.slides = [] # reveal slide export of notebooks
self.material = [] # additional material (pictures, etc.)
self.chapter_zips = []
self.executed_notebooks = [self.get_executed_notebook(notebook)
for notebook in self.notebooks]
self.zip_file = self.conf['output_path'] / \
Path(self.conf['slug_title']).with_suffix('.zip')
# Instantiate notebook exporter
self.html_exporter = HTMLExporter()
# Instantiate slide exporter
self.slide_exporter = SlidesExporter()
reveal_url = \
"https://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.7.0"
if self.conf['local_reveal']:
self.slide_exporter.reveal_url_prefix = 'reveal.js'
else:
self.slide_exporter.reveal_url_prefix = reveal_url
def _get_pages(self):
"""get a list of target html pages from markdown source"""
# Start html_pages list with index.html
html_pages = [self.conf['output_path'] /
self.conf['pages']['home']['name']]
for md_page in self.md_pages:
html_pages.append(self.conf['output_path'] /
md_page.relative_to(
self.conf['pages']['path']).with_suffix('.html'))
return html_pages
@staticmethod
def _get_config(config_file):
def _get_user_config(config_file: Path) -> dict:
"""
Parse yaml file to build the corresponding configuration dictionary
"""
with open(config_file, 'r') as f:
config = yaml.safe_load(f)
config['configpath'] = Path(os.path.dirname(
os.path.abspath(config_file)))
return config
return yaml.safe_load(f)
def build_pages(self):
"""
......@@ -174,52 +125,199 @@ class NbCourse:
children = parent['children']
for child in children.values():
# Render markdown pages
child_page = MarkdownPage(child['html'],
child_page = MarkdownPage(self, child['html'],
title=child['title'],
src=child['md'],
parent=parent_page)
child_page.render(self.config)
child_page.render()
render_children(child, child_page)
except KeyError:
# Parent page has no children
return
# First build homepage
try:
homepage_filename = self.config['pages']['home']['name']
except KeyError:
homepage_filename = 'index.html'
# Render homepage
homepage = HomePage(homepage_filename)
homepage.render(self.config)
homepage = HomePage(self)
homepage.render()
# Then build children pages
render_children(self.config['pages']['home'], homepage)
render_children(self.conf['pages']['home'], homepage)
def build_book(self, pdf_title=DEFAULT_PDF_TITLE):
def build_book(self):
"""Build a pdf book using bookbook"""
logging.basicConfig(level=logging.INFO)
# Render basic bookbook template including /pages/titlepage.tex content
full_template_path = self.config['configpath'] / TEMPLATE_PATH
env = jinja2.Environment(loader=jinja2.FileSystemLoader(
full_template_path.as_posix()))
template = env.get_template('book.tplx')
titlepage_path = self.config['configpath'] / \
Path("pages/titlepage.tex")
with open(titlepage_path) as f:
titlepage_content = f.read()
bookbook_template = template.render(titlepage=titlepage_content)
with open(BOOKBOOK_FILENAME, 'w') as f:
f.write(bookbook_template)
# Call bookbook
combine_and_convert(Path('.'),
Path(pdf_title),
pdf=True,
template_file=BOOKBOOK_FILENAME)
def build(self):
self.build_pages()
self.build_book()
book = Book(self.conf)
book.build()
def task_output_dir(self):
"""Create empty output directory"""
return {
'targets': [self.conf['output_path']],
'actions': [(create_folder, [self.conf['output_path']])],
'uptodate': [self.conf['output_path'].is_dir],
'clean': True,
}
def get_src_dst_paths(self, src_path: Path):
"""Return a tuple of file paths for source and destination"""
files = get_file_list(src_path)
src_files = [src_path / file for file in files]
dst_path = self.conf['output_path'] / Path(src_path.name)
dst_files = [dst_path / file for file in files]
return src_files, dst_path, dst_files
def task_copy_material(self):
"""Copy notebook and theme material to output directory"""
for parent in 'nb', 'theme':
for src_path in self.conf[parent]['material_paths']:
src_files, dst_path, dst_files = self.get_src_dst_paths(
src_path)
self.material += dst_files
yield {
'name': dst_path,
'file_dep': src_files,
'targets': dst_files,
'actions': [(update_material, (src_path, dst_path))],
'clean': [(clean_tree, (dst_path,))],
}
def task_copy_reveal(self):
"""Copy reveal.js to output directory"""
if self.conf['local_reveal']:
src_path = self.conf['reveal_path']
src_files, dst_path, dst_files = self.get_src_dst_paths(src_path)
return {
'file_dep': src_files,
'targets': dst_files,
'actions': [(update_material, (src_path, dst_path))],
'clean': [(clean_tree, (dst_path,))],
}
def execute_notebook(self, notebook: Path, executed_notebook: Path):
"""Execute notebook and write result to executed_notebook file"""
with open(notebook) as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(timeout=60,
kernel_name='python3',
allow_errors=True)
ep.preprocess(nb, {'metadata': {'path': self.conf['nb']['path']}})
with open(executed_notebook, 'w', encoding='utf-8') as f:
nbformat.write(nb, f)
def get_executed_notebook(self, notebook) -> Path:
"""Return executed notebook path"""
return self.conf['output_path'] / Path(notebook.name)
def task_execute_notebooks(self):
"""Write executed notebooks to output directory"""
for notebook in self.notebooks:
executed_notebook = self.get_executed_notebook(notebook)
yield {
'name': executed_notebook,
'file_dep': [notebook],
'targets': [executed_notebook],
'clean': True,
'actions': [(self.execute_notebook, (notebook,
executed_notebook))],
}
def convert_notebook(self, notebook: Path,
output_file: Path,
format='html'):
"""Convert notebook to output_file"""
if format == 'slide':
exporter = self.slide_exporter
else:
exporter = self.html_exporter
with open(notebook) as f:
nb = nbformat.read(f, as_version=4)
body, resources = exporter.from_notebook_node(nb)
with open(output_file, 'w', encoding='utf-8') as f:
f.write(body)
def task_convert_to_html(self):
"""Convert executed notebook to html page"""
for notebook in self.notebooks:
executed_notebook = self.get_executed_notebook(notebook)